From d2bcb58ffb83ac4feb84f315b4c3ea7ee2f94ca1 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Sun, 22 Jan 2012 01:16:13 -0700 Subject: [PATCH 001/598] The runas feature that was added in 93423aa2e5e4b7de6452090b0039560d2b1302e4 jacks up Windows support for the minion. Adding in these 2 try statements allows Windows to load the minion. --- salt/modules/cmd.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/salt/modules/cmd.py b/salt/modules/cmd.py index de30ce2f7b8a..6aa3eecd4ce2 100644 --- a/salt/modules/cmd.py +++ b/salt/modules/cmd.py @@ -10,7 +10,10 @@ import subprocess import tempfile import salt.utils -import pwd +try: + import pwd +except: + pass # Set up logging log = logging.getLogger(__name__) @@ -33,8 +36,11 @@ def _run(cmd, Do the DRY thing and only call subprocess.Popen() once ''' ret = {} - uid = os.getuid() - euid = os.geteuid() + try: + uid = os.getuid() + euid = os.geteuid() + except: + pass def su(): os.setuid(runas_uid) From 215e7b616eb5a97f6a5e6999dfd19e6e24252c98 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 6 Feb 2012 19:10:08 -0600 Subject: [PATCH 002/598] Clarify the docstring for "managed" and "recurse" --- salt/states/file.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 0722d175f74e..9d1ba7d184ef 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -401,7 +401,7 @@ def managed(name, The location of the file to manage source - The source file to download to the minion, this source file can be + The source file to download to the minion. This source file can be hosted on either the salt master server, or on an http or ftp server. For files hosted on the salt file server, if the file is located on the master in the directory named spam, and is called eggs, the source @@ -411,7 +411,7 @@ def managed(name, If the file is hosted on a http or ftp server then the source_hash argument is also required - source_hash: + source_hash This can be either a file which contains a source hash string for the source, or a source hash string. The source hash string is the hash algorithm followed by the hash of the file: @@ -444,6 +444,11 @@ def managed(name, defaults Default context passed to the template. + + __env__ + Specify a file environment from file_roots. Defaults to "base". Note + that you must use the double underscores in the option name in your sls + files. ''' if mode: mode = str(mode) @@ -879,7 +884,7 @@ def recurse(name, over to the specified path. name - The directory to set the recursion in + The directory in which to set the recursion source The source directory, this directory is located on the salt master file @@ -891,6 +896,11 @@ def recurse(name, Make sure that only files that are set up by salt and required by this function are kept. If this option is set then everything in this directory will be deleted unless it is required. + + __env__ + Specify a file environment from file_roots. Defaults to "base". Note + that you must use the double underscores in the option name in your sls + files. ''' ret = {'name': name, 'changes': {}, From 8822a10abf8bf4018b38a626619244b1ea1386b3 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 6 Feb 2012 21:38:06 -0600 Subject: [PATCH 003/598] Revert "Clarify the docstring for "managed" and "recurse"" This reverts commit 215e7b616eb5a97f6a5e6999dfd19e6e24252c98. --- salt/states/file.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 9d1ba7d184ef..0722d175f74e 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -401,7 +401,7 @@ def managed(name, The location of the file to manage source - The source file to download to the minion. This source file can be + The source file to download to the minion, this source file can be hosted on either the salt master server, or on an http or ftp server. For files hosted on the salt file server, if the file is located on the master in the directory named spam, and is called eggs, the source @@ -411,7 +411,7 @@ def managed(name, If the file is hosted on a http or ftp server then the source_hash argument is also required - source_hash + source_hash: This can be either a file which contains a source hash string for the source, or a source hash string. The source hash string is the hash algorithm followed by the hash of the file: @@ -444,11 +444,6 @@ def managed(name, defaults Default context passed to the template. - - __env__ - Specify a file environment from file_roots. Defaults to "base". Note - that you must use the double underscores in the option name in your sls - files. ''' if mode: mode = str(mode) @@ -884,7 +879,7 @@ def recurse(name, over to the specified path. name - The directory in which to set the recursion + The directory to set the recursion in source The source directory, this directory is located on the salt master file @@ -896,11 +891,6 @@ def recurse(name, Make sure that only files that are set up by salt and required by this function are kept. If this option is set then everything in this directory will be deleted unless it is required. - - __env__ - Specify a file environment from file_roots. Defaults to "base". Note - that you must use the double underscores in the option name in your sls - files. ''' ret = {'name': name, 'changes': {}, From 93d1ac09fe58ca2a4c0c16e9973d41c3fc9063b4 Mon Sep 17 00:00:00 2001 From: ixtix Date: Wed, 15 Feb 2012 14:49:56 +0100 Subject: [PATCH 004/598] added kwarg grant_option for adding GRANT OPTION(http://http://dev.mysql.com/doc/refman/5.0/en/grant.html) to modules/mysql and states/mysql_grant --- salt/modules/mysql.py | 23 ++++++++++++++++------- salt/states/mysql_grants.py | 14 ++++++++------ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index a1a8182515f5..d1eb2f19ec53 100755 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -512,7 +512,8 @@ def db_optimize(name, def __grant_generate(grant, database, user, - host='localhost'): + host='localhost', + grant_option=False): # todo: Re-order the grant so it is according to the SHOW GRANTS for xxx@yyy query (SELECT comes first, etc) grant = grant.replace(',', ', ').upper() @@ -521,6 +522,8 @@ def __grant_generate(grant, table = db_part[2] query = "GRANT %s ON `%s`.`%s` TO '%s'@'%s'" % (grant, db, table, user, host,) + if grant_option: + query += " WITH GRANT OPTION" log.debug("Query generated: {0}".format(query,)) return query @@ -553,10 +556,11 @@ def user_grants(user, def grant_exists(grant, database, user, - host='localhost'): + host='localhost', + grant_option=False): # todo: This function is a bit tricky, since it requires the ordering to be exactly the same. # perhaps should be replaced/reworked with a better/cleaner solution. - target = __grant_generate(grant, database, user, host) + target = __grant_generate(grant, database, user, host, grant_option) if target in user_grants(user, host): log.debug("Grant exists.") @@ -568,7 +572,8 @@ def grant_exists(grant, def grant_add(grant, database, user, - host='localhost'): + host='localhost', + grant_option=False): ''' Adds a grant to the MySQL server. @@ -580,7 +585,7 @@ def grant_add(grant, db = connect() cur = db.cursor() - query = __grant_generate(grant, database, user, host) + query = __grant_generate(grant, database, user, host, grant_option) log.debug("Query: {0}".format(query,)) if cur.execute( query ): log.info("Grant '{0}' created") @@ -590,7 +595,8 @@ def grant_add(grant, def grant_revoke(grant, database, user, - host='localhost'): + host='localhost', + grant_option=False): ''' Removes a grant from the MySQL server. @@ -600,7 +606,10 @@ def grant_revoke(grant, ''' # todo: validate grant db = connect() - cur = db.cursor() + cur = db.cursor() + + if grant_option: + grant += ", GRANT OPTION" query = "REVOKE %s ON %s FROM '%s'@'%s';" % (grant, database, user, host,) log.debug("Query: {0}".format(query,)) if cur.execute( query ): diff --git a/salt/states/mysql_grants.py b/salt/states/mysql_grants.py index f9931625b39e..7f7369ccf6e1 100755 --- a/salt/states/mysql_grants.py +++ b/salt/states/mysql_grants.py @@ -43,7 +43,8 @@ def present(name, grant=None, database=None, user=None, - host='localhost'): + host='localhost', + grant_option=False): ''' Ensure that the grant is present with the specified properties @@ -73,11 +74,11 @@ def present(name, ) } # check if grant exists - if __salt__['mysql.grant_exists'](grant,database,user,host): + if __salt__['mysql.grant_exists'](grant, database, user, host, grant_option): return ret # The grant is not present, make it! - if __salt__['mysql.grant_add'](grant,database,user,host): + if __salt__['mysql.grant_add'](grant, database, user, host, grant_option): ret['comment'] = 'Grant {0} for {1}@{2} on {3} has been added'.format( grant, user, @@ -100,7 +101,8 @@ def absent(name, grant=None, database=None, user=None, - host='localhost'): + host='localhost', + grant_option=False): ''' Ensure that the grant is absent @@ -125,8 +127,8 @@ def absent(name, 'comment': ''} #check if db exists and remove it - if __salt__['mysql.grant_exists'](grant,database,user,host,): - if __salt__['mysql.grant_revoke'](grant,database,user,host): + if __salt__['mysql.grant_exists'](grant, database, user, host, grant_option): + if __salt__['mysql.grant_revoke'](grant, database, user, host, grant_option): ret['comment'] = ('Grant {0} for {1}@{2} on {3} has been' ' revoked').format( grant, From 09d1dc43a1fa431067e7a0330852bb03c3d31ce7 Mon Sep 17 00:00:00 2001 From: ixtix Date: Wed, 15 Feb 2012 16:16:10 +0100 Subject: [PATCH 005/598] Added escape flag to mysql_grant, default: True In states: mysql_grant.present/absent and in modules: mysql.grant_exists/grant_add/__grant_generate there is now a escape option. This let's you handle the escaping of the database value in the grants by your self. --- salt/modules/mysql.py | 20 +++++++++++++------- salt/states/mysql_grants.py | 18 +++++++++++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index d1eb2f19ec53..9339adf567cb 100755 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -513,15 +513,19 @@ def __grant_generate(grant, database, user, host='localhost', - grant_option=False): + grant_option=False, + escape=True): # todo: Re-order the grant so it is according to the SHOW GRANTS for xxx@yyy query (SELECT comes first, etc) grant = grant.replace(',', ', ').upper() db_part = database.rpartition('.') db = db_part[0] table = db_part[2] - - query = "GRANT %s ON `%s`.`%s` TO '%s'@'%s'" % (grant, db, table, user, host,) + + if escape: + db = "`%s`" % db + table = "`%s`" % table + query = "GRANT %s ON %s.%s TO '%s'@'%s'" % (grant, db, table, user, host,) if grant_option: query += " WITH GRANT OPTION" log.debug("Query generated: {0}".format(query,)) @@ -557,10 +561,11 @@ def grant_exists(grant, database, user, host='localhost', - grant_option=False): + grant_option=False, + escape=True): # todo: This function is a bit tricky, since it requires the ordering to be exactly the same. # perhaps should be replaced/reworked with a better/cleaner solution. - target = __grant_generate(grant, database, user, host, grant_option) + target = __grant_generate(grant, database, user, host, grant_option, escape) if target in user_grants(user, host): log.debug("Grant exists.") @@ -573,7 +578,8 @@ def grant_add(grant, database, user, host='localhost', - grant_option=False): + grant_option=False, + escape=True): ''' Adds a grant to the MySQL server. @@ -585,7 +591,7 @@ def grant_add(grant, db = connect() cur = db.cursor() - query = __grant_generate(grant, database, user, host, grant_option) + query = __grant_generate(grant, database, user, host, grant_option, escape) log.debug("Query: {0}".format(query,)) if cur.execute( query ): log.info("Grant '{0}' created") diff --git a/salt/states/mysql_grants.py b/salt/states/mysql_grants.py index 7f7369ccf6e1..410fc6cfa06d 100755 --- a/salt/states/mysql_grants.py +++ b/salt/states/mysql_grants.py @@ -44,7 +44,8 @@ def present(name, database=None, user=None, host='localhost', - grant_option=False): + grant_option=False, + escape=True): ''' Ensure that the grant is present with the specified properties @@ -62,6 +63,12 @@ def present(name, host The MySQL server + + grant_option + Adds the WITH GRANT OPTION to the defined grant. default: False + + excape + Defines if the database value gets escaped or not. default: True ''' ret = {'name': name, 'changes': {}, @@ -74,11 +81,11 @@ def present(name, ) } # check if grant exists - if __salt__['mysql.grant_exists'](grant, database, user, host, grant_option): + if __salt__['mysql.grant_exists'](grant, database, user, host, grant_option, escape): return ret # The grant is not present, make it! - if __salt__['mysql.grant_add'](grant, database, user, host, grant_option): + if __salt__['mysql.grant_add'](grant, database, user, host, grant_option, escape): ret['comment'] = 'Grant {0} for {1}@{2} on {3} has been added'.format( grant, user, @@ -102,7 +109,8 @@ def absent(name, database=None, user=None, host='localhost', - grant_option=False): + grant_option=False, + escape=True): ''' Ensure that the grant is absent @@ -127,7 +135,7 @@ def absent(name, 'comment': ''} #check if db exists and remove it - if __salt__['mysql.grant_exists'](grant, database, user, host, grant_option): + if __salt__['mysql.grant_exists'](grant, database, user, host, grant_option, escape): if __salt__['mysql.grant_revoke'](grant, database, user, host, grant_option): ret['comment'] = ('Grant {0} for {1}@{2} on {3} has been' ' revoked').format( From 66671143978fd2044fe73d27152fa0ffde5bea96 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 15 Feb 2012 10:51:15 -0700 Subject: [PATCH 006/598] remove faulty log entry, we don't have a logger in here --- salt/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/__init__.py b/salt/__init__.py index dba74aa306e3..4b0f52656f5a 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -30,9 +30,7 @@ def set_pidfile(pidfile): try: open(pidfile, 'w+').write(str(os.getpid())) except IOError: - err = ('Failed to commit the pid file to location {0}, please verify' - ' that the location is available').format(pidfile) - log.error(err) + pass def verify_env(dirs): From 21ba494638f19a48eb6260e6e704073c8c01c5e8 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 15 Feb 2012 11:16:58 -0700 Subject: [PATCH 007/598] Make the release a little clearer with the proc dir --- doc/topics/releases/0.9.7.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/releases/0.9.7.rst b/doc/topics/releases/0.9.7.rst index 9a4b4fb976ec..b0f27ac2a6ae 100644 --- a/doc/topics/releases/0.9.7.rst +++ b/doc/topics/releases/0.9.7.rst @@ -28,7 +28,7 @@ system allows clear introspection into the active running state of the, running Salt interface. The Jobs interface is centered in the new minion side proc system. The -minions now store a msgpack serialized file under ``cachedir``/proc. These +minions now store a msgpack serialized file under /var/cache/salt/proc. These files keep track of the active state of processes on the minion. Functions in the saltutil Module From 06ef45f5d4bd217fa012fee89df036fe15dc8d4d Mon Sep 17 00:00:00 2001 From: Shane Hathaway Date: Wed, 15 Feb 2012 22:06:27 -0700 Subject: [PATCH 008/598] Fix for issue #704, msgpack list/tuple conversion leads to silent failures and other problems --- salt/payload.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/payload.py b/salt/payload.py index c1a1dc9aa2d9..3a5b19c9059c 100644 --- a/salt/payload.py +++ b/salt/payload.py @@ -20,7 +20,7 @@ def unpackage(package_): ''' Unpackages a payload ''' - return msgpack.loads(package_) + return msgpack.loads(package_, use_list=True) def format_payload(enc, **kwargs): @@ -49,12 +49,12 @@ def loads(self, msg): Run the correct loads serialization format ''' if self.serial == 'msgpack': - return msgpack.loads(msg) + return msgpack.loads(msg, use_list=True) elif self.serial == 'pickle': try: return pickle.loads(msg) except: - return msgpack.loads(msg) + return msgpack.loads(msg, use_list=True) def load(self, fn_): ''' From 43da63534b62795e5418bd4c4c1d8d042ad3438b Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 16 Feb 2012 09:59:34 -0700 Subject: [PATCH 009/598] add check for recursive requisites for __id__ --- salt/state.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 2cc54463ff03..67f3b1273cf4 100644 --- a/salt/state.py +++ b/salt/state.py @@ -235,7 +235,8 @@ def verify_data(self, data): if reqdec: for req in data[reqdec]: if data['state'] == req.keys()[0]: - if data['name'] == req[req.keys()[0]]: + if data['name'] == req[req.keys()[0]] \ + or data['__id__'] == req[req.keys()[0]]: err = ('Recursive require detected in SLS {0} for' ' require {1} in ID {2}').format( data['__sls__'], From d517bd9b98f2cfe35092bcaed9db53ac74807520 Mon Sep 17 00:00:00 2001 From: elfixit Date: Fri, 17 Feb 2012 14:50:35 +0100 Subject: [PATCH 010/598] added password_hash option to mysql.user with the password_hash option it is now possible to provide directly a hashed password instad of a plaintext password. also fixed some indents in the files, there where some with just 3 blanks --- salt/modules/mysql.py | 72 ++++++++++++++++++++++----------------- salt/states/mysql_user.py | 9 ++--- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index 9339adf567cb..3b05f4878387 100755 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -206,7 +206,7 @@ def db_list(): cur.execute('SHOW DATABASES') results = cur.fetchall() for dbs in results: - ret.append(dbs[0]) + ret.append(dbs[0]) log.debug(ret) return ret @@ -220,8 +220,8 @@ def db_tables(name): salt '*' mysql.db_tables 'database' ''' if not db_exists(name): - log.info("Database '{0}' does not exist".format(name,)) - return False + log.info("Database '{0}' does not exist".format(name,)) + return False ret = [] db = connect() @@ -232,7 +232,7 @@ def db_tables(name): cur.execute(query) results = cur.fetchall() for table in results: - ret.append(table[0]) + ret.append(table[0]) log.debug(ret) return ret @@ -251,7 +251,7 @@ def db_exists(name): cur.execute( query ) result_set = cur.fetchall() if cur.rowcount == 1: - return True + return True return False @@ -274,8 +274,8 @@ def db_create(name): query = "CREATE DATABASE `%s`;" % name log.debug("Query: {0}".format(query,)) if cur.execute( query ): - log.info("DB '{0}' created".format(name,)) - return True + log.info("DB '{0}' created".format(name,)) + return True return False def db_remove(name): @@ -303,8 +303,8 @@ def db_remove(name): cur.execute( query ) if not db_exists(name): - log.info("Database '{0}' has been removed".format(name,)) - return True + log.info("Database '{0}' has been removed".format(name,)) + return True log.info("Database '{0}' has not been removed".format(name,)) return False @@ -342,7 +342,7 @@ def user_exists(user, log.debug("Doing query: {0}".format(query,)) cur.execute( query ) if cur.rowcount == 1: - return True + return True return False def user_info(user, @@ -365,7 +365,8 @@ def user_info(user, def user_create(user, host='localhost', - password=None): + password=None, + password_hash=None): ''' Creates a MySQL user. @@ -381,21 +382,24 @@ def user_create(user, cur = db.cursor () query = "CREATE USER '%s'@'%s'" % (user, host,) if password is not None: - query = query + " IDENTIFIED BY '%s'" % password + query = query + " IDENTIFIED BY '%s'" % password + elif password_hash is not None: + query = query + " IDENTIFIED BY PASSWORD '%'" % password_hash log.debug("Query: {0}".format(query,)) cur.execute( query ) if user_exists(user,host): - log.info("User '{0}'@'{1}' has been created".format(user,host,)) - return True + log.info("User '{0}'@'{1}' has been created".format(user,host,)) + return True log.info("User '{0}'@'{1}' is not created".format(user,host,)) return False def user_chpass(user, host='localhost', - password=None): + password=None, + password_hash=None): ''' Change password for MySQL user @@ -403,17 +407,21 @@ def user_chpass(user, salt '*' mysql.user_chpass frank localhost newpassword ''' - if password is None: - log.error('No password provided') - return False + if password is None or password_hash is None: + log.error('No password provided') + return False + elif password is not None: + password_sql = "PASSWORD(\"%s\")" % password + elif password_hash is not None: + password_sql = "\"%s\"" % password_hash db = connect() cur = db.cursor () - query = "UPDATE mysql.user SET password=PASSWORD(\"%s\") WHERE User='%s' AND Host = '%s';" % (password,user,host,) + query = "UPDATE mysql.user SET password=%s WHERE User='%s' AND Host = '%s';" % (password_sql,user,host,) log.debug("Query: {0}".format(query,)) if cur.execute( query ): - log.info("Password for user '{0}'@'{1}' has been changed".format(user,host,)) - return True + log.info("Password for user '{0}'@'{1}' has been changed".format(user,host,)) + return True log.info("Password for user '{0}'@'{1}' is not changed".format(user,host,)) return False @@ -434,8 +442,8 @@ def user_remove(user, cur.execute(query) result = cur.fetchone() if not user_exists(user,host): - log.info("User '{0}'@'{1}' has been removed".format(user,host,)) - return True + log.info("User '{0}'@'{1}' has been removed".format(user,host,)) + return True log.info("User '{0}'@'{1}' has NOT been removed".format(user,host,)) return False @@ -483,7 +491,7 @@ def db_repair(name, else: log.info("Repairing table '%s' in db '%s'..".format(name,table,)) ret = __repair_table(name,table) - return ret + return ret def db_optimize(name, table=None): @@ -529,7 +537,7 @@ def __grant_generate(grant, if grant_option: query += " WITH GRANT OPTION" log.debug("Query generated: {0}".format(query,)) - return query + return query def user_grants(user, host='localhost'): @@ -553,7 +561,7 @@ def user_grants(user, cur.execute(query) results = cur.fetchall() for grant in results: - ret.append(grant[0]) + ret.append(grant[0]) log.debug(ret) return ret @@ -569,10 +577,10 @@ def grant_exists(grant, if target in user_grants(user, host): log.debug("Grant exists.") - return True + return True log.debug("Grant does not exist, or is perhaps not ordered properly?") - return False + return False def grant_add(grant, database, @@ -594,8 +602,8 @@ def grant_add(grant, query = __grant_generate(grant, database, user, host, grant_option, escape) log.debug("Query: {0}".format(query,)) if cur.execute( query ): - log.info("Grant '{0}' created") - return True + log.info("Grant '{0}' created") + return True return False def grant_revoke(grant, @@ -619,7 +627,7 @@ def grant_revoke(grant, query = "REVOKE %s ON %s FROM '%s'@'%s';" % (grant, database, user, host,) log.debug("Query: {0}".format(query,)) if cur.execute( query ): - log.info("Grant '{0}' revoked") - return True + log.info("Grant '{0}' revoked") + return True return False diff --git a/salt/states/mysql_user.py b/salt/states/mysql_user.py index cc0fe0dcd32f..81f09aba941e 100644 --- a/salt/states/mysql_user.py +++ b/salt/states/mysql_user.py @@ -15,7 +15,8 @@ def present(name, host='localhost', - password=None): + password=None, + password_hash=None): ''' Ensure that the named user is present with the specified properties @@ -28,10 +29,10 @@ def present(name, 'comment': 'User {0}@{1} is already present'.format(name,host,)} # check if user exists if __salt__['mysql.user_exists'](name,host): - return ret + return ret # The user is not present, make it! - if __salt__['mysql.user_create'](name,host,password,): + if __salt__['mysql.user_create'](name,host,password,password_hash): ret['comment'] = 'The user {0}@{1} has been added'.format(name,host,) ret['changes'][name] = 'Present' else: @@ -60,7 +61,7 @@ def absent(name, ret['comment'] = 'User {0}@{1} has been removed'.format(name,host,) ret['changes'][name] = 'Absent' return ret - + # fallback ret['comment'] = 'User {0}@{1} is not present, so it cannot be removed'.format(name,host,) return ret From 1311e007a17ad3133fc2d7629bc5ae14075d4085 Mon Sep 17 00:00:00 2001 From: elfixit Date: Fri, 17 Feb 2012 15:04:01 +0100 Subject: [PATCH 011/598] fixed stupid spell mistake --- salt/modules/mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index 3b05f4878387..862fac1c0eff 100755 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -384,7 +384,7 @@ def user_create(user, if password is not None: query = query + " IDENTIFIED BY '%s'" % password elif password_hash is not None: - query = query + " IDENTIFIED BY PASSWORD '%'" % password_hash + query = query + " IDENTIFIED BY PASSWORD '%s'" % password_hash log.debug("Query: {0}".format(query,)) cur.execute( query ) From a45ba616dfceb5d076ca31e0538afb798f2ed095 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 17 Feb 2012 11:41:48 -0700 Subject: [PATCH 012/598] write reffernce on requisites --- doc/ref/states/ordering.rst | 171 ++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) diff --git a/doc/ref/states/ordering.rst b/doc/ref/states/ordering.rst index b5b269b2977d..a1ab1a088c8d 100644 --- a/doc/ref/states/ordering.rst +++ b/doc/ref/states/ordering.rst @@ -14,6 +14,177 @@ consist of requisite declarations and order options. Salt does **not** execute :term:`state declarations ` in the order they appear in the source. +Requisite Statements +==================== + +.. note:: + + This document represents behavior exhibited by Salt requisites as of + version 0.9.7 of Salt. + +Often when setting up states any single action will require or depend on +another action. Salt allows you to build relationships between states with +requisite statements. A requisite statement ensure that the named state is +evaluated before the state requiring it. There are two types of requisite +statements in Salt, **require** and **watch**. + +These requisite statements are applied to a specific state declaration: + +.. code-block:: yaml + + httpd: + pkg: + - installed + file: + - managed + - name: /etc/httpd/conf/httpd.conf + - source: salt://httpd/httpd.conf + - require: + - pkg: httpd + +In this example we use the **require** requisite to declare that the file +/etc/httpd/conf/httpd.conf should only be set up if the pkg state executes +successfully. + +The requisite system works by finding the states that are required and +executing them before the state that requires them. Then the required states +can be evaluated to see if they have executed correctly. + +Multiple Requisites +------------------- + +The requisite statement is passed as a list, allowing for the easy addition of +more requisites. Both requisite types can also be separately declared: + +.. code-block:: yaml + + httpd: + pkg: + - installed + service: + - running + - enable: True + - watch: + - file: /etc/httpd/conf/httpd.conf + - require: + - pkg: httpd + - user: httpd + - group: httpd + file: + - managed + - name: /etc/httpd/conf/httpd.conf + - source: salt://httpd/httpd.conf + - require: + - pkg: httpd + user: + - present + group: + - present + +In this example the httpd service is only going to be started if the package, +user, group and file are executed successfully. + +The Require Requisite +--------------------- + +The foundation of the requisite system is the ``require`` requisite. The +require requisite ensures that the required state(s) are executed before the +requiring state. So, if a state is declared that sets down a vimrc, then it +would be pertinent to make sure that the vimrc file would only be set down if +the vim package has been installed: + +.. code-block:: yaml + + vim: + pkg: + - installed + file: + - managed + - source: salt://vim/vimrc + - require: + - pkg: vim + +In this case, the vimrc file will only be applied by Salt if and after the vim +package is installed. + +The Watch Requisite +------------------- + +The ``watch`` requisite is more advanced than the ``require`` requisite. The +watch requisite executes the same logic as require (therefore if something is +watched it does not need to also be required) with the addition of executing +logic if the required states have changed in some way. + +The watch requisite checks to see if the watched states have returned any +changes. If the watched state returns changes, and the watched states execute +successfully, then the watching state will execute a function that reacts to +the changes in the watched states. + +Perhaps an example can better explain the behavior: + +.. code-block:: yaml + + redis: + pkg: + - latest + file: + - managed + - source: salt://redis/redis.conf + - name: /etc/redis.conf + - require: + - pkg: redis + service: + - running + - enable: True + - watch: + - file: /etc/redis.conf + - pkg: redis + +In this example the redis service will only be started if the file +/etc/redis.conf is applied, and the file is only applied if the package is +installed. This is normal require behavior, but if the watched file changes, +or the watched package is installed or upgraded, then the redis service is +restarted. + +Watch and the Watcher Function +------------------------------ + +The watch requisite is based on the ``watcher`` function, state python +modules can include a function called watcher, this function is then called +if the watch call is invoked. In the case of the service module the underlying +service is restarted. In the case of the cmd state the command is executed. + +The watcher function for the service state looks like this: + +.. code-block:: python + + def watcher(name, sig=None): + ''' + The service watcher, called to invoke the watch command. + + name + The name of the init or rc script used to manage the service + + sig + The string to search for when looking for the service process with ps + ''' + if __salt__['service.status'](name, sig): + changes = {name: __salt__['service.restart'](name)} + return {'name': name, + 'changes': changes, + 'result': True, + 'comment': 'Service restarted'} + + return {'name': name, + 'changes': {}, + 'result': True, + 'comment': 'Service {0} started'.format(name)} + +The watch requisite only works if the state that is watching has a watcher +function written. If watch is set on a state that does not have a watcher +function (like pkg), then the listed states will behave only as if they were +under a ``require`` statement. + The Order Option ================ From bb58d17c8be4e676587dfb675f8afa2a34e98c03 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 17 Feb 2012 12:52:29 -0700 Subject: [PATCH 013/598] Add more docs to the state writing document --- doc/ref/states/writing.rst | 73 +++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/doc/ref/states/writing.rst b/doc/ref/states/writing.rst index f98f4af6e54e..955922269a05 100644 --- a/doc/ref/states/writing.rst +++ b/doc/ref/states/writing.rst @@ -35,7 +35,10 @@ Using Custom State Modules ========================== Place your custom state modules inside a ``_states`` directory within the -``file_roots`` specified by the master config file. +``file_roots`` specified by the master config file. These custome state modules +can then be distributed in a number of ways. Custom state modules are +distributed when state.highstate is run, or via the saltutil.sync_states +function. Cross Calling Modules ===================== @@ -69,3 +72,71 @@ A State Module must return a dict containing the following keys/values: - **result:** A boolean value. *True* if the action was successful, otherwise *False*. - **comment:** A string containing a summary of the result. + +Watcher Function +================ + +If the state being written should support the watch requisite then a watcher +function needs to be declared. The watcher function is called whenever the +watch requisite is invoked and should be generic to the behavior of the state +itself. + +The watcher function should accept all of the options that the normal state +functions accept (as they will be passed into the watcher function). + +A watcher function typically is used to execute state specific reactive +behavior, for instance, the watcher for the service module restarts the +named service and makes it useful for the watcher to make the service +react to changes in the environment. + +The watcher function also needs to return the same data that a normal state +function returns. + + +Mod_init Interface +================== + +Some states need to execute something only once to ensure that an environment +has been set up, or certain conditions global to the state behavior can be +predefined. This is the realm of the mod_init interface. + +A state module can have a function called **mod_init** which executes when the +first state of this type is called. This interface was created primarily to +improve the pkg state. When packages are installed the package metadata needs +to be refreshed, but refreshing the package metadata every time a package is +installed is wasteful. The mod_init function for the pkg state sets a flag down +so that the first, and only the first, package installation attempt will refresh +the package database (the package database can of course be manually called to +refresh via the ``refresh`` option in the pkg state). + +The mod_init function must accept the **Low State Data** for the given +executing state as an argument. The low state data is a dict and can be seen by +executing the state.show_lowstate function. Then the mod_init function must +return a bool. If the return value is True, then the mod_init function will not +be executed again, meaning that the needed behavior has been set up. Otherwise, +if the mod_init function returns False, then the function will be called the +next time. + +A good example of the mod_init function is found in the pkg state module: + +.. code-block:: python + + def mod_init(low): + ''' + Refresh the package database here so that it only needs to happen once + ''' + if low['fun'] == 'installed' or low['fun'] == 'latest': + rtag = __gen_rtag() + if not os.path.exists(rtag): + open(rtag, 'w+').write('') + return True + else: + return False + +The mod_init function in the pkg state accepts the low state data as ``low`` +and then checks to see if the function being called is going to install +packages, if the function is not going to install packages then there is no +need to refresh the package database. Therefore if the package database is +prepared to refresh, then return True and the mod_init will not be called +the next time a pkg state is evaluated, otherwise return False and the mod_init +will be called next time a pkg state is evaluated. From 454d9156f3b1dca9c0cddddee64abee0bf5deae1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 17 Feb 2012 13:08:31 -0700 Subject: [PATCH 014/598] add module doc for the sys module --- doc/ref/modules/sys.rst | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 doc/ref/modules/sys.rst diff --git a/doc/ref/modules/sys.rst b/doc/ref/modules/sys.rst new file mode 100644 index 000000000000..89f6491b8a73 --- /dev/null +++ b/doc/ref/modules/sys.rst @@ -0,0 +1,37 @@ +===================== +The sys Pseudo Module +===================== + +The regular salt modules execute in a separate context from the salt minion +and manipulating the actual salt modules needs to happen in a higher level +context within the minion process. This is where the sys pseudo module is +used. + +The sys pseudo module comes with a few functions that return data about the +available functions on the minion or allows for the minion modules to be +refreshed. These functions are as follows: + +sys.list_functions +================== + +Return a list of all the loaded and available functions on the specified +minion + +sys.list_modules +================ + +Return a list of all the loaded and available modules on the specified +minion + +sys.doc +======= + +This meta function combines the documentation data from the selected minion +modules and displays it to the terminal in a clean, readable way. + +sys.reload_modules +================== + +The reload_modules function invokes a reload of all of the modules on a +minion. This function can be called if the modules need to be re-evaluated for +availability or new modules have been made available to the minion. From b15642c7da64b9bc9b6bd5ff9f8d269ad3980d1b Mon Sep 17 00:00:00 2001 From: Eivind Uggedal Date: Fri, 17 Feb 2012 21:14:40 +0100 Subject: [PATCH 015/598] More idiomatic building of dicts with list or dict values. --- salt/state.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/salt/state.py b/salt/state.py index 67f3b1273cf4..6825b3be832e 100644 --- a/salt/state.py +++ b/salt/state.py @@ -16,6 +16,7 @@ import logging import os import tempfile +from collections import defaultdict import salt.loader import salt.minion @@ -781,9 +782,9 @@ def get_tops(self): ''' Gather the top files ''' - tops = {} - include = {} - done = {} + tops = defaultdict(list) + include = defaultdict(list) + done = defaultdict(list) # Gather initial top files if self.opts['environment']: tops[self.opts['environment']] = [ @@ -797,8 +798,6 @@ def get_tops(self): ] else: for env in self._get_envs(): - if not env in tops: - tops[env] = [] tops[env].append( self.state.compile_template( self.client.cache_file( @@ -814,8 +813,6 @@ def get_tops(self): for ctop in ctops: if not 'include' in ctop: continue - if not env in include: - include[env] = [] for sls in ctop['include']: include[env].append(sls) ctop.pop('include') @@ -823,8 +820,6 @@ def get_tops(self): while include: pops = [] for env, states in include.items(): - if not env in done: - done[env] = [] pops.append(env) if not states: continue @@ -851,14 +846,12 @@ def merge_tops(self, tops): ''' Cleanly merge the top files ''' - top = {} + top = defaultdict(dict) for sourceenv, ctops in tops.items(): for ctop in ctops: for env, targets in ctop.items(): if env == 'include': continue - if not env in top: - top[env] = {} for tgt in targets: if not tgt in top[env]: top[env][tgt] = ctop[env][tgt] From 2a550ebd3f765d9896021b721be9bb0f38a07833 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 17 Feb 2012 13:48:36 -0700 Subject: [PATCH 016/598] add a tmp dir to tests --- tests/tmp/_README | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/tmp/_README diff --git a/tests/tmp/_README b/tests/tmp/_README new file mode 100644 index 000000000000..f06e2f9f7aa6 --- /dev/null +++ b/tests/tmp/_README @@ -0,0 +1,5 @@ +======= +TMP DIR +======= +The tmp dir is used to place files that need to be modified durring the +execution of the test suite From f86203f26e0e1e9c657a4f272a5a8f71a4306636 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 17 Feb 2012 13:48:47 -0700 Subject: [PATCH 017/598] fix broken tests due to the msgpack list fix --- tests/modules/hosts.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/modules/hosts.py b/tests/modules/hosts.py index d8371a159af0..b64fa9226fc6 100644 --- a/tests/modules/hosts.py +++ b/tests/modules/hosts.py @@ -40,8 +40,8 @@ def test_list_hosts(self): self.__clean_hosts() hosts = self.run_function('hosts.list_hosts') self.assertEqual(len(hosts), 6) - self.assertEqual(hosts['::1'], ('ip6-localhost', 'ip6-loopback')) - self.assertEqual(hosts['127.0.0.1'], ('localhost', 'myname')) + self.assertEqual(hosts['::1'], ['ip6-localhost', 'ip6-loopback']) + self.assertEqual(hosts['127.0.0.1'], ['localhost', 'myname']) def test_list_hosts_nofile(self): ''' @@ -68,10 +68,10 @@ def test_get_alias(self): hosts.get_alias ''' self.__clean_hosts() - self.assertEqual(self.run_function('hosts.get_alias', ['127.0.0.1']), ('localhost', 'myname')) - self.assertEqual(self.run_function('hosts.get_alias', ['127.0.0.2']), ()) + self.assertEqual(self.run_function('hosts.get_alias', ['127.0.0.1']), ['localhost', 'myname']) + self.assertEqual(self.run_function('hosts.get_alias', ['127.0.0.2']), []) self.__clear_hosts() - self.assertEqual(self.run_function('hosts.get_alias', ['127.0.0.1']), ()) + self.assertEqual(self.run_function('hosts.get_alias', ['127.0.0.1']), []) def test_has_pair(self): ''' From 514faf931439370860efbc5bfe9530d1e4dba256 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 17 Feb 2012 14:04:05 -0700 Subject: [PATCH 018/598] initial troubleshooting doc --- doc/topics/troubleshooting/index.rst | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 doc/topics/troubleshooting/index.rst diff --git a/doc/topics/troubleshooting/index.rst b/doc/topics/troubleshooting/index.rst new file mode 100644 index 000000000000..571a633818ac --- /dev/null +++ b/doc/topics/troubleshooting/index.rst @@ -0,0 +1,33 @@ +=============== +Troubleshooting +=============== + +The intent of the troubleshootign secion is to introduce solutions to a +number of common issues encountered by users and the tools that are available +to aid in developing states and salt code. + +Running in the Foreground +========================= + +A great deal of information is available via the debug logging system, if you +are having issues with minions connecting or not starting run the minion and/or +master in the foreground: + + # salt-master -l debug + # salt-minion -l debug + + +Using salt-call +=============== + +The salt-call command was originally developed for aiding in the development +of new salt modules. Since then many applications have arisen for the salt-call +command that is bundled with the salt minion. These range from the original +intent of the salt-call, developmetn assistance, to gathering large amounts of +data from complex calls like state.highstate. + +When developing the state tree it is geenrally recommended to invoke +state.highstate with salt-call, this displays a great deal more information +about the highstate execution then if it is called remotely. + + From d5e4b4a94bedd48e24c976af3b4dad0bdf549701 Mon Sep 17 00:00:00 2001 From: Seth House Date: Fri, 17 Feb 2012 14:44:49 -0700 Subject: [PATCH 019/598] Moved some of new sys module documentation into the existing doc --- doc/ref/modules/all/salt.modules.sys.rst | 13 +++++++-- doc/ref/modules/sys.rst | 37 ------------------------ 2 files changed, 11 insertions(+), 39 deletions(-) delete mode 100644 doc/ref/modules/sys.rst diff --git a/doc/ref/modules/all/salt.modules.sys.rst b/doc/ref/modules/all/salt.modules.sys.rst index 4cd2f4230a76..45be5b94e9c0 100644 --- a/doc/ref/modules/all/salt.modules.sys.rst +++ b/doc/ref/modules/all/salt.modules.sys.rst @@ -2,7 +2,14 @@ salt.modules.sys ================ -A pseudo-module for working with modules on a minion. +The regular salt modules execute in a separate context from the salt minion +and manipulating the actual salt modules needs to happen in a higher level +context within the minion process. This is where the sys pseudo module is +used. + +The sys pseudo module comes with a few functions that return data about the +available functions on the minion or allows for the minion modules to be +refreshed. These functions are as follows: .. py:module:: salt.modules.sys @@ -13,7 +20,9 @@ A pseudo-module for working with modules on a minion. .. py:function:: reload_modules - Instruct the minion to reload all available modules in memory. + Instruct the minion to reload all available modules in memory. This + function can be called if the modules need to be re-evaluated for + availability or new modules have been made available to the minion. .. py:function:: list_modules diff --git a/doc/ref/modules/sys.rst b/doc/ref/modules/sys.rst deleted file mode 100644 index 89f6491b8a73..000000000000 --- a/doc/ref/modules/sys.rst +++ /dev/null @@ -1,37 +0,0 @@ -===================== -The sys Pseudo Module -===================== - -The regular salt modules execute in a separate context from the salt minion -and manipulating the actual salt modules needs to happen in a higher level -context within the minion process. This is where the sys pseudo module is -used. - -The sys pseudo module comes with a few functions that return data about the -available functions on the minion or allows for the minion modules to be -refreshed. These functions are as follows: - -sys.list_functions -================== - -Return a list of all the loaded and available functions on the specified -minion - -sys.list_modules -================ - -Return a list of all the loaded and available modules on the specified -minion - -sys.doc -======= - -This meta function combines the documentation data from the selected minion -modules and displays it to the terminal in a clean, readable way. - -sys.reload_modules -================== - -The reload_modules function invokes a reload of all of the modules on a -minion. This function can be called if the modules need to be re-evaluated for -availability or new modules have been made available to the minion. From dd993edf5d1da9368e3ba355d8f2c1de4d718f22 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 17 Feb 2012 16:33:22 -0700 Subject: [PATCH 020/598] Repair issue where existing grain was not overwritten with update --- salt/minion.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/minion.py b/salt/minion.py index d42a85cb1020..dad92d3afeae 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -117,6 +117,21 @@ def __load_modules(self): ''' Return the functions and the returners loaded up from the loader module ''' + pre_opts = {} + salt.config.load_config( + pre_opts, + self.opts['conf_file'], + 'SALT_MINION_CONFIG' + ) + if 'include' in pre_opts: + pre_opts = salt.config.include_config( + pre_opts, + self.opts['conf_file'] + ) + if 'grains' in pre_opts: + self.opts['grains'] = pre_opts['grains'] + else: + self.opts['grains'] = {} self.opts['grains'] = salt.loader.grains(self.opts) functions = salt.loader.minion_mods(self.opts) returners = salt.loader.returners(self.opts) From c5af26f31a6399b63c6f219cb36ef45bc75c16f1 Mon Sep 17 00:00:00 2001 From: Seth House Date: Sat, 18 Feb 2012 10:40:50 -0700 Subject: [PATCH 021/598] Playing with RTD tweak to use Sphinx default theme --- doc/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/conf.py b/doc/conf.py index c86a657177e8..ba41cf5fc584 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -107,6 +107,7 @@ def __getattr__(self, name): ### HTML options html_theme = 'default' +html_style = None html_title = None html_short_title = 'Salt' From c03716e470aa1863bf422cb4e882a11c0a56f38f Mon Sep 17 00:00:00 2001 From: Seth House Date: Sat, 18 Feb 2012 10:43:59 -0700 Subject: [PATCH 022/598] Revert "Playing with RTD tweak to use Sphinx default theme" This reverts commit c5af26f31a6399b63c6f219cb36ef45bc75c16f1. --- doc/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index ba41cf5fc584..c86a657177e8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -107,7 +107,6 @@ def __getattr__(self, name): ### HTML options html_theme = 'default' -html_style = None html_title = None html_short_title = 'Salt' From b7a96a47a64c1616f474f1257b46964a4f174ca1 Mon Sep 17 00:00:00 2001 From: Seth House Date: Sat, 18 Feb 2012 11:33:56 -0700 Subject: [PATCH 023/598] Changed the "In depth" sections into real headings --- doc/index.rst | 193 ++++++++++++++++++++++++++------------------------ 1 file changed, 101 insertions(+), 92 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index 613b4e27186f..80756cb8df49 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -56,132 +56,141 @@ Setting up and using Salt is a simple task but it's capabilities run much, much deeper. Gaining a better understanding of how Salt works will allow you to truly make it work for you. -.. contents:: Overview +.. contents:: The components of Salt :local: :depth: 2 -**Remote execution** - Remote execution is the core functionality of Salt. Running pre-defined or - arbitrary commands on remote hosts. +Remote execution +---------------- - **Modules** - Salt modules are the core of remote execution. They provide - functionality such as installing a package, restarting a service, - running a remote command, transferring a file — and the list goes on. +Remote execution is the core functionality of Salt. Running pre-defined or +arbitrary commands on remote hosts. - :doc:`Full list of modules ` - The giant list of core modules that ship with Salt - (And there are even more in the `salt-contrib`_ repository!) +**Modules** + Salt modules are the core of remote execution. They provide + functionality such as installing a package, restarting a service, + running a remote command, transferring a file — and the list goes on. - :doc:`Writing modules ` - A guide on how to write Salt modules + :doc:`Full list of modules ` + The giant list of core modules that ship with Salt + (And there are even more in the `salt-contrib`_ repository!) - **Targeting** - Specify which hosts should run commands or manage configuration. + :doc:`Writing modules ` + A guide on how to write Salt modules - :doc:`Targeting ` - Hostnames, lists, regular expressions, or define groups. +**Targeting** + Specify which hosts should run commands or manage configuration. - :doc:`Grains ` - Bits of static information about a minion such as OS, version, - virtualization, CPU, memory, and much more. + :doc:`Targeting ` + Hostnames, lists, regular expressions, or define groups. - **Returners** - Salt returners allow saving minion responses in various datastores or - to various locations in addition to display at the CLI. + :doc:`Grains ` + Bits of static information about a minion such as OS, version, + virtualization, CPU, memory, and much more. - :doc:`Full list of returners ` - Store minion responses in Redis, Mongo, Cassandra or more. +**Returners** + Salt returners allow saving minion responses in various datastores or + to various locations in addition to display at the CLI. - :doc:`Writing returners ` - If we're missing your favorite storage backend, webservice, or you - need a custom endpoint returners are *tiny* and simple to write. + :doc:`Full list of returners ` + Store minion responses in Redis, Mongo, Cassandra or more. -**Configuration management** - Building on the remote execution core is a robust and flexible config - management framework. Execution happens on the minions allowing - effortless, simultaneous configuration of thousands of hosts. + :doc:`Writing returners ` + If we're missing your favorite storage backend, webservice, or you + need a custom endpoint returners are *tiny* and simple to write. - **States** - Express the state of a host using small, easy to read, easy to - understand configuration files. No programming required (unless you - want to). +Configuration management +------------------------ + +Building on the remote execution core is a robust and flexible config +management framework. Execution happens on the minions allowing +effortless, simultaneous configuration of thousands of hosts. + +**States** + Express the state of a host using small, easy to read, easy to + understand configuration files. No programming required (unless you + want to). + + :doc:`Full list of states ` + Install packages, create users, transfer files, start services, and + more and more. + + :doc:`Using states ` + You've seen the big list of available states, now learn how to call + them. - :doc:`Full list of states ` - Install packages, create users, transfer files, start services, and - more and more. + :doc:`Highstate data structure ` + A dry, vocabulary and technical representation of the configuration + format that states represent. - :doc:`Using states ` - You've seen the big list of available states, now learn how to call - them. +**Renderers** + Write state configuration files in the language, templating engine, or + file type of your choice. The world doesn't need yet another DSL. - :doc:`Highstate data structure ` - A dry, vocabulary and technical representation of the configuration - format that states represent. + :doc:`Full list of renderers ` + YAML? JSON? Jinja? Mako? Python? We got you covered. (And if we + don't, new renderers are *tiny* and easy to write.) - **Renderers** - Write state configuration files in the language, templating engine, or - file type of your choice. The world doesn't need yet another DSL. + :doc:`Renderers ` + Salt states are only concerned with the ultimate highstate data + structure. How you create that data structure isn't our business. + Tweak a config option and use whatever you're most comfortable + with. - :doc:`Full list of renderers ` - YAML? JSON? Jinja? Mako? Python? We got you covered. (And if we - don't, new renderers are *tiny* and easy to write.) +Miscellaneous topics +-------------------- - :doc:`Renderers ` - Salt states are only concerned with the ultimate highstate data - structure. How you create that data structure isn't our business. - Tweak a config option and use whatever you're most comfortable - with. +Salt is a many splendid thing. -**Miscellaneous topics** - Salt is a many splendid thing. +:doc:`File Server ` + Salt can easily and quickly transfer files (in fact, that's how Salt + States work). Even under load, files are chunked and served. - :doc:`File Server ` - Salt can easily and quickly transfer files (in fact, that's how Salt - States work). Even under load, files are chunked and served. +:doc:`Syndic ` + A seamless master of masters. Scale Salt to thousands of hosts or + across many different networks. - :doc:`Syndic ` - A seamless master of masters. Scale Salt to thousands of hosts or - across many different networks. +:doc:`Peer communication ` + Allow minions to communicate amongst themselves. For example, configure + one minion by querying live data from all the others. With great power + comes great responsibility. - :doc:`Peer communication ` - Allow minions to communicate amongst themselves. For example, configure - one minion by querying live data from all the others. With great power - comes great responsibility. +:doc:`Network topology ` + At it's core, Salt is a highly scalable communication layer built on + top of ZeroMQ that enables remote execution and configuration + management. The possibilities are endless and Salt's future looks + bright. - :doc:`Network topology ` - At it's core, Salt is a highly scalable communication layer built on - top of ZeroMQ that enables remote execution and configuration - management. The possibilities are endless and Salt's future looks - bright. +:doc:`Python API interface ` + Use Salt programmatically from your own scripts and programs easily and + simply via ``import salt``. - :doc:`Python API interface ` - Use Salt programmatically from your own scripts and programs easily and - simply via ``import salt``. +Reference +--------- -**Reference** - :doc:`Command-line interface ` - Read the Salt manpages. +:doc:`Command-line interface ` + Read the Salt manpages. - :doc:`Full list of master settings ` - Read through the heavily-commented master configuration file. +:doc:`Full list of master settings ` + Read through the heavily-commented master configuration file. - :doc:`Full list of minion settings ` - Read through the heavily-commented minion configuration file. +:doc:`Full list of minion settings ` + Read through the heavily-commented minion configuration file. - :doc:`Full table of contents ` - Dense but complete. +:doc:`Full table of contents ` + Dense but complete. -**More information about the project** +More information about the project +---------------------------------- - :doc:`Roadmap ` - Where we're headed. +:doc:`Roadmap ` + Where we're headed. - :doc:`Release notes ` - Where we've been. +:doc:`Release notes ` + Where we've been. - :doc:`Community ` - How you can get involved. +:doc:`Community ` + How you can get involved. .. _`salt-contrib`: https://github.com/saltstack/salt-contrib .. _`salt-states`: https://github.com/saltstack/salt-states From 46c280167ff060f37481a4ef4bb302c0e713a644 Mon Sep 17 00:00:00 2001 From: Seth House Date: Sat, 18 Feb 2012 11:35:15 -0700 Subject: [PATCH 024/598] Added links to other EC2 and FreeBSD tutorials --- doc/index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/index.rst b/doc/index.rst index 80756cb8df49..e3e2c925d6d2 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -56,6 +56,11 @@ Setting up and using Salt is a simple task but it's capabilities run much, much deeper. Gaining a better understanding of how Salt works will allow you to truly make it work for you. +.. sidebar:: More tutorials! + + * :doc:`Bootstraping Salt on EC2 ` + * :doc:`Installing Salt on FreeBSD ` + .. contents:: The components of Salt :local: :depth: 2 From 14a6c598c5e365d6201e466460e123cb056a1229 Mon Sep 17 00:00:00 2001 From: Seth House Date: Sat, 18 Feb 2012 11:41:12 -0700 Subject: [PATCH 025/598] Added class to screencast icon to make it display better --- doc/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/index.rst b/doc/index.rst index e3e2c925d6d2..607421d42bda 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,6 +1,7 @@ .. _contents: .. |vid| image:: /_static/film_link.png + :class: math Get started with Salt ===================== From 7d4e07b13844385628f483365e2dde7bbe32e554 Mon Sep 17 00:00:00 2001 From: Seth House Date: Sat, 18 Feb 2012 11:48:02 -0700 Subject: [PATCH 026/598] Add more template overrides to link back to the homepage RTD uses layout.html internally so you can't override blocks directly from it. --- doc/_templates/domainindex.html | 6 ++++++ doc/_templates/page.html | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 doc/_templates/domainindex.html create mode 100644 doc/_templates/page.html diff --git a/doc/_templates/domainindex.html b/doc/_templates/domainindex.html new file mode 100644 index 000000000000..44d04d5f7b23 --- /dev/null +++ b/doc/_templates/domainindex.html @@ -0,0 +1,6 @@ +{% extends "!domainindex.html" %} + +{%- block rootrellink %} +
  • « SaltStack.org | 
    • +
  • Documentation home
    • +{%- endblock %} diff --git a/doc/_templates/page.html b/doc/_templates/page.html new file mode 100644 index 000000000000..f8d09ebe8c30 --- /dev/null +++ b/doc/_templates/page.html @@ -0,0 +1,6 @@ +{% extends "!page.html" %} + +{%- block rootrellink %} +
  • « SaltStack.org | 
    • +
  • Documentation home
    • +{%- endblock %} From ba0ebb2389c062b08873c2f77b5cfae6e0fd6682 Mon Sep 17 00:00:00 2001 From: Eivind Uggedal Date: Sun, 19 Feb 2012 09:35:35 +0100 Subject: [PATCH 027/598] Use in operator to check for existence in set. --- salt/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 6825b3be832e..615997d7efe0 100644 --- a/salt/state.py +++ b/salt/state.py @@ -966,7 +966,7 @@ def render_state(self, sls, env, mods): errors.append(err) else: for sub_sls in state.pop('include'): - if not list(mods).count(sub_sls): + if sub_sls not in mods: nstate, mods, err = self.render_state( sub_sls, env, From ebfe4048c76f1c3a74fa6e79599f8c9f92f16a1d Mon Sep 17 00:00:00 2001 From: Eivind Uggedal Date: Sun, 19 Feb 2012 14:26:55 +0100 Subject: [PATCH 028/598] 'key' in str/list reads better than str/list.count('key'). --- salt/cli/__init__.py | 2 +- salt/cli/key.py | 6 +++--- salt/client.py | 4 ++-- salt/minion.py | 2 +- salt/modules/kvm_hyper.py | 1 - salt/modules/mdadm.py | 4 ++-- salt/modules/mount.py | 4 ++-- salt/modules/pw_user.py | 2 +- salt/modules/useradd.py | 2 +- salt/modules/virt.py | 5 +---- salt/state.py | 2 +- salt/states/file.py | 5 ++--- salt/states/kmod.py | 2 +- 13 files changed, 18 insertions(+), 23 deletions(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index b82318274f10..116666abd817 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -177,7 +177,7 @@ def __parse(self): else: opts['tgt'] = args[0] - if args[1].count(','): + if ',' in args[1]: opts['fun'] = args[1].split(',') opts['arg'] = [] for comp in ' '.join(args[2:]).split(','): diff --git a/salt/cli/key.py b/salt/cli/key.py index 6e13a84b89bb..98a97e76c0d5 100644 --- a/salt/cli/key.py +++ b/salt/cli/key.py @@ -123,7 +123,7 @@ def _accept(self, key): minions_pre, minions_rejected) = self._check_minions_directories() pre = os.listdir(minions_pre) - if not pre.count(key): + if key not in pre: err = ('The key named %s does not exist, please accept an ' 'available key' %(key)) #log.error(err) @@ -174,7 +174,7 @@ def _reject(self, key): minions_pre, minions_rejected) = self._check_minions_directories() pre = os.listdir(minions_pre) - if not pre.count(key): + if key not in pre: err = ('The host named %s is unavailable, please accept an ' 'available key' %(key)) self._log(err, level='error') @@ -235,4 +235,4 @@ def run(self): elif self.opts['delete']: self._delete_key() else: - self._list_all() \ No newline at end of file + self._list_all() diff --git a/salt/client.py b/salt/client.py index 9e835b5981bb..bee273603f55 100644 --- a/salt/client.py +++ b/salt/client.py @@ -102,8 +102,8 @@ def _check_list_minions(self, expr): ''' ret = [] for fn_ in os.listdir(os.path.join(self.opts['pki_dir'], 'minions')): - if expr.count(fn_): - if not ret.count(fn_): + if fn_ in expr: + if fn_ not in ret: ret.append(fn_) return ret diff --git a/salt/minion.py b/salt/minion.py index dad92d3afeae..d2f7a2586b54 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -992,7 +992,7 @@ def get_state(self, sls, env): Get a state file from the master and store it in the local minion cache return the location of the file ''' - if sls.count('.'): + if '.' in sls: sls = sls.replace('.', '/') for path in ['salt://' + sls + '.sls', os.path.join('salt://', sls, 'init.sls')]: diff --git a/salt/modules/kvm_hyper.py b/salt/modules/kvm_hyper.py index ab1204e9b815..c44712e1a7c0 100644 --- a/salt/modules/kvm_hyper.py +++ b/salt/modules/kvm_hyper.py @@ -40,7 +40,6 @@ def __virtual__(): return False if 'kvm_' not in open('/proc/modules').read(): return False - #libvirt_ret = __salt__['cmd.run'](__grains__['ps']).count('libvirtd') try: libvirt_conn = libvirt.open('qemu:///system') libvirt_conn.close() diff --git a/salt/modules/mdadm.py b/salt/modules/mdadm.py index 0c4501db832f..3c7c9df7310d 100644 --- a/salt/modules/mdadm.py +++ b/salt/modules/mdadm.py @@ -28,7 +28,7 @@ def list(): ret = {} for line in (__salt__['cmd.run_stdout'] ('mdadm --detail --scan').split('\n')): - if not line.count(' '): + if ' ' not in line: continue comps = line.split() metadata = comps[2].split('=') @@ -57,7 +57,7 @@ def detail(device='/dev/md0'): for line in __salt__['cmd.run_stdout'](cmd).split('\n'): if line.startswith(device): continue - if not line.count(' '): + if ' ' not in line: continue if not ':' in line: if '/dev/' in line: diff --git a/salt/modules/mount.py b/salt/modules/mount.py index ee183fc3d43b..d7b1fa43e196 100644 --- a/salt/modules/mount.py +++ b/salt/modules/mount.py @@ -217,7 +217,7 @@ def remount(name, device, mkmnt=False, fstype='', opts='defaults'): mnts = active() if name in mnts: # The mount point is mounted, attempt to remount it with the given data - if not opts.count('remount'): + if 'remount' not in opts: opts.append('remount') lopts = ','.join(opts) cmd = 'mount -o {0} {1} {2} '.format(lopts, device, name) @@ -246,6 +246,6 @@ def is_fuse_exec(cmd): continue out = __salt__['cmd.run']('ldd {0}'.format(path)) for line in out.split('\n'): - if line.count('libfuse'): + if 'libfuse' in line: return True return False diff --git a/salt/modules/pw_user.py b/salt/modules/pw_user.py index 6a1d0a0e84ac..ee613f8f3fc2 100644 --- a/salt/modules/pw_user.py +++ b/salt/modules/pw_user.py @@ -214,6 +214,6 @@ def list_groups(name): ''' ugrp = set() for group in grp.getgrall(): - if group.gr_mem.count(name): + if name in group.gr_mem: ugrp.add(group.gr_name) return sorted(list(ugrp)) diff --git a/salt/modules/useradd.py b/salt/modules/useradd.py index f617269af510..7467dcd48495 100644 --- a/salt/modules/useradd.py +++ b/salt/modules/useradd.py @@ -225,7 +225,7 @@ def list_groups(name): ugrp = set() for group in grp.getgrall(): - if group.gr_mem.count(name): + if name in group.gr_mem: ugrp.add(group.gr_name) return sorted(list(ugrp)) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 1584bba80bda..6ecc47b7ce7d 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -511,7 +511,4 @@ def is_kvm_hyper(): return False if 'kvm_' not in open('/proc/modules').read(): return False - libvirt_ret = __salt__['cmd.run'](__grains__['ps']).count('libvirtd') - if not libvirt_ret: - return False - return True + return 'libvirtd' in __salt__['cmd.run'](__grains__['ps']) diff --git a/salt/state.py b/salt/state.py index 615997d7efe0..4120bd22768d 100644 --- a/salt/state.py +++ b/salt/state.py @@ -824,7 +824,7 @@ def get_tops(self): if not states: continue for sls in states: - if done[env].count(sls): + if sls in done[env]: continue tops[env].append( self.state.compile_template( diff --git a/salt/states/file.py b/salt/states/file.py index 4acdde354b35..7be41063da69 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -117,9 +117,8 @@ def _is_bin(path): Return True if a file is a bin, just checks for NULL char, this should be expanded to reflect how git checks for bins ''' - if open(path, 'rb').read(2048).count('\0'): - return True - return False + with open(path, 'rb') as f: + return '\0' in f.read(2048) def _gen_keep_files(name, require): diff --git a/salt/states/kmod.py b/salt/states/kmod.py index 5ed5a9d913a4..33a10a83abca 100644 --- a/salt/states/kmod.py +++ b/salt/states/kmod.py @@ -33,7 +33,7 @@ def present(name): .format(name)) return ret # Module is not loaded, verify availability - if not __salt__['kmod.available']().count(name): + if name not in __salt__['kmod.available'](): ret['comment'] = 'Kernel module {0} is unavailable'.format(name) ret['result'] = False return ret From 6c60042121bee1987979dbf13381728cbf236ff7 Mon Sep 17 00:00:00 2001 From: Eivind Uggedal Date: Sun, 19 Feb 2012 15:16:30 +0100 Subject: [PATCH 029/598] Return boolean conditional instead of using an if block with `return True` inside. No need to explicitly return True inside a truthy if block. --- salt/modules/ebuild.py | 5 +---- salt/modules/freebsdkmod.py | 5 +---- salt/modules/gentoo_service.py | 8 ++------ salt/modules/groupadd.py | 3 +-- salt/modules/hosts.py | 6 +----- salt/modules/kvm_hyper.py | 10 ++-------- salt/modules/mysql.py | 8 ++------ salt/modules/pw_group.py | 3 +-- salt/modules/pw_user.py | 16 +++++----------- salt/modules/rh_service.py | 8 ++------ salt/modules/shadow.py | 4 +--- salt/modules/solr.py | 9 ++------- salt/modules/useradd.py | 16 +++++----------- salt/modules/virt.py | 10 ++-------- salt/modules/win_useradd.py | 6 ++---- salt/state.py | 3 +-- salt/states/pkg.py | 3 +-- 17 files changed, 32 insertions(+), 91 deletions(-) diff --git a/salt/modules/ebuild.py b/salt/modules/ebuild.py index bca005942560..64de8a168393 100644 --- a/salt/modules/ebuild.py +++ b/salt/modules/ebuild.py @@ -73,10 +73,7 @@ def refresh_db(): salt '*' pkg.refresh_db ''' - if __salt__['cmd.retcode']('emerge --sync --quiet'): - return False - else: - return True + return __salt__['cmd.retcode']('emerge --sync --quiet') == 0 def install(pkg, refresh=False, **kwargs): ''' diff --git a/salt/modules/freebsdkmod.py b/salt/modules/freebsdkmod.py index 6fb2e9f4b0b3..f680e6ce87f4 100644 --- a/salt/modules/freebsdkmod.py +++ b/salt/modules/freebsdkmod.py @@ -66,10 +66,7 @@ def check_available(mod): salt '*' kmod.check_available kvm ''' - if mod in available(): - # the module is available, return True - return True - return False + return mod in available() def lsmod(): diff --git a/salt/modules/gentoo_service.py b/salt/modules/gentoo_service.py index 8b4b7fc4494e..3c1d14bb0e94 100644 --- a/salt/modules/gentoo_service.py +++ b/salt/modules/gentoo_service.py @@ -146,9 +146,7 @@ def enabled(name): salt '*' service.enabled ''' - if name in get_enabled(): - return True - return False + return name in get_enabled() def disabled(name): ''' @@ -158,6 +156,4 @@ def disabled(name): salt '*' service.enabled ''' - if name in get_disabled(): - return True - return False + return name in get_disabled() diff --git a/salt/modules/groupadd.py b/salt/modules/groupadd.py index 530172716f87..8bb3f62acefc 100644 --- a/salt/modules/groupadd.py +++ b/salt/modules/groupadd.py @@ -87,6 +87,5 @@ def chgid(name, gid): __salt__['cmd.run'](cmd) post_gid = __salt__['file.group_to_gid'](name) if post_gid != pre_gid: - if post_gid == gid: - return True + return post_gid == gid return False diff --git a/salt/modules/hosts.py b/salt/modules/hosts.py index f639328f734f..32d984707d27 100644 --- a/salt/modules/hosts.py +++ b/salt/modules/hosts.py @@ -83,11 +83,7 @@ def has_pair(ip, alias): salt '*' hosts.has_pair ''' hosts = list_hosts() - if ip not in hosts: - return False - if alias in hosts[ip]: - return True - return False + return ip in hosts and alias in hosts[ip] def set_host(ip, alias): diff --git a/salt/modules/kvm_hyper.py b/salt/modules/kvm_hyper.py index c44712e1a7c0..815de126f5f7 100644 --- a/salt/modules/kvm_hyper.py +++ b/salt/modules/kvm_hyper.py @@ -394,16 +394,10 @@ def set_autostart(name): dom = _get_dom(name) if state == 'on': - if dom.setAutostart(1) == 0: - return True - else: - return False + return dom.setAutostart(1) == 0 elif state == 'off': - if dom.setAutostart(0) == 0: - return True - else: - return False + return dom.setAutostart(0) == 0 else: # return False if state is set to something other then on or off diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index 862fac1c0eff..95e12773774a 100755 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -250,9 +250,7 @@ def db_exists(name): log.debug("Doing query: {0}".format(query,)) cur.execute( query ) result_set = cur.fetchall() - if cur.rowcount == 1: - return True - return False + return cur.rowcount == 1 def db_create(name): @@ -341,9 +339,7 @@ def user_exists(user, query = "SELECT User,Host FROM mysql.user WHERE User = '%s' AND Host = '%s'" % (user, host,) log.debug("Doing query: {0}".format(query,)) cur.execute( query ) - if cur.rowcount == 1: - return True - return False + return cur.rowcount == 1 def user_info(user, host='localhost'): diff --git a/salt/modules/pw_group.py b/salt/modules/pw_group.py index 1a05af1a6504..a787beb6f6ce 100644 --- a/salt/modules/pw_group.py +++ b/salt/modules/pw_group.py @@ -86,6 +86,5 @@ def chgid(name, gid): __salt__['cmd.run'](cmd) post_gid = __salt__['file.group_to_gid'](name) if post_gid != pre_gid: - if post_gid == gid: - return True + return post_gid == gid return False diff --git a/salt/modules/pw_user.py b/salt/modules/pw_user.py index ee613f8f3fc2..5cd8e7eb70af 100644 --- a/salt/modules/pw_user.py +++ b/salt/modules/pw_user.py @@ -91,8 +91,7 @@ def chuid(name, uid): __salt__['cmd.run'](cmd) post_info = info(name) if post_info['uid'] != pre_info['uid']: - if post_info['uid'] == uid: - return True + return post_info['uid'] == uid return False @@ -111,8 +110,7 @@ def chgid(name, gid): __salt__['cmd.run'](cmd) post_info = info(name) if post_info['gid'] != pre_info['gid']: - if post_info['gid'] == gid: - return True + return post_info['gid'] == gid return False @@ -131,8 +129,7 @@ def chshell(name, shell): __salt__['cmd.run'](cmd) post_info = info(name) if post_info['shell'] != pre_info['shell']: - if post_info['shell'] == shell: - return True + return post_info['shell'] == shell return False @@ -155,8 +152,7 @@ def chhome(name, home, persist=False): __salt__['cmd.run'](cmd) post_info = info(name) if post_info['home'] != pre_info['home']: - if post_info['home'] == home: - return True + return post_info['home'] == home return False @@ -179,9 +175,7 @@ def chgroups(name, groups, append=False): cmd += '-a' __salt__['cmd.run'](cmd) agrps = set(list_groups(name)) - if ugrps.difference(agrps): - return True - return False + return len(ugrps - agrps) == 0 def info(name): diff --git a/salt/modules/rh_service.py b/salt/modules/rh_service.py index 12e1d6dd7847..be00c38c49e5 100644 --- a/salt/modules/rh_service.py +++ b/salt/modules/rh_service.py @@ -163,9 +163,7 @@ def enabled(name): salt '*' service.enabled ''' - if name in get_enabled(): - return True - return False + return name in get_enabled() def disabled(name): @@ -176,6 +174,4 @@ def disabled(name): salt '*' service.disabled ''' - if name in get_disabled(): - return True - return False + return name in get_disabled() diff --git a/salt/modules/shadow.py b/salt/modules/shadow.py index 0ff3907f92ac..318e9425127f 100644 --- a/salt/modules/shadow.py +++ b/salt/modules/shadow.py @@ -63,6 +63,4 @@ def set_password(name, password): lines.append('{0}\n'.format(line)) open(s_file, 'w+').writelines(lines) uinfo = info(name) - if uinfo['pwd'] == password: - return True - return False + return uinfo['pwd'] == password diff --git a/salt/modules/solr.py b/salt/modules/solr.py index d70d997708f9..0226215ddf09 100644 --- a/salt/modules/solr.py +++ b/salt/modules/solr.py @@ -132,10 +132,7 @@ def _check_for_cores(): True if one or more cores defined in __opts__['solr.cores'] ''' - if len(__opts__['solr.cores']) > 0: - return True - else: - return False + return len(__opts__['solr.cores']) > 0 def _get_return_dict(success=True, data={}, errors=[], warnings=[]): ''' @@ -313,9 +310,7 @@ def _is_master(): Return: boolean:: True if __opts__['solr.type'] = master ''' - if __opts__['solr.type'] == 'master': - return True - return False + return __opts__['solr.type'] == 'master' def _merge_options(options): ''' diff --git a/salt/modules/useradd.py b/salt/modules/useradd.py index 7467dcd48495..7561548fb551 100644 --- a/salt/modules/useradd.py +++ b/salt/modules/useradd.py @@ -92,8 +92,7 @@ def chuid(name, uid): __salt__['cmd.run'](cmd) post_info = info(name) if post_info['uid'] != pre_info['uid']: - if post_info['uid'] == uid: - return True + return post_info['uid'] == uid return False @@ -112,8 +111,7 @@ def chgid(name, gid): __salt__['cmd.run'](cmd) post_info = info(name) if post_info['gid'] != pre_info['gid']: - if post_info['gid'] == gid: - return True + return post_info['gid'] == gid return False @@ -132,8 +130,7 @@ def chshell(name, shell): __salt__['cmd.run'](cmd) post_info = info(name) if post_info['shell'] != pre_info['shell']: - if post_info['shell'] == shell: - return True + return post_info['shell'] == shell return False @@ -156,8 +153,7 @@ def chhome(name, home, persist=False): __salt__['cmd.run'](cmd) post_info = info(name) if post_info['home'] != pre_info['home']: - if post_info['home'] == home: - return True + return post_info['home'] == home return False @@ -180,9 +176,7 @@ def chgroups(name, groups, append=False): cmd += '-a' __salt__['cmd.run'](cmd) agrps = set(list_groups(name)) - if ugrps.difference(agrps): - return True - return False + return len(ugrps - agrps) == 0 def info(name): diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 6ecc47b7ce7d..12d708de4fa0 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -417,16 +417,10 @@ def set_autostart(vm_, state='on'): dom = _get_dom(vm_) if state == 'on': - if dom.setAutostart(1) == 0: - return True - else: - return False + return dom.setAutostart(1) == 0 elif state == 'off': - if dom.setAutostart(0) == 0: - return True - else: - return False + return dom.setAutostart(0) == 0 else: # return False if state is set to something other then on or off diff --git a/salt/modules/win_useradd.py b/salt/modules/win_useradd.py index 392a58e77384..03b3b1053777 100644 --- a/salt/modules/win_useradd.py +++ b/salt/modules/win_useradd.py @@ -108,8 +108,7 @@ def chhome(name, home): __salt__['cmd.run'](cmd) post_info = info(name) if post_info['home'] != pre_info['home']: - if post_info['home'] == home: - return True + return post_info['home'] == home return False @@ -130,8 +129,7 @@ def chprofile(name, profile): __salt__['cmd.run'](cmd) post_info = info(name) if post_info['profile'] != pre_info['profile']: - if post_info['profile'] == profile: - return True + return post_info['profile'] == profile return False def info(name): diff --git a/salt/state.py b/salt/state.py index 4120bd22768d..57a5eba6e7fa 100644 --- a/salt/state.py +++ b/salt/state.py @@ -571,8 +571,7 @@ def check_failhard(self, low, running): if low.get('failhard', False) \ or self.opts['failhard'] \ and tag in running: - if not running[tag]['result']: - return True + return not running[tag]['result'] return False def check_requisite(self, low, running, chunks): diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 9e6cfb7d6862..421510a88298 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -205,8 +205,7 @@ def mod_init(low): if not os.path.exists(rtag): open(rtag, 'w+').write('') return True - else: - return False + return False def __gen_rtag(): ''' From 8a2138ff4052b68027f0fede708a5f6be7df0ea0 Mon Sep 17 00:00:00 2001 From: Eivind Uggedal Date: Sun, 19 Feb 2012 15:31:17 +0100 Subject: [PATCH 030/598] Favor the clearer set - set syntax in stead of set.difference(set) --- salt/modules/freebsdkmod.py | 4 ++-- salt/modules/freebsdservice.py | 5 ++--- salt/modules/kmod.py | 4 ++-- salt/modules/saltutil.py | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/salt/modules/freebsdkmod.py b/salt/modules/freebsdkmod.py index f680e6ce87f4..7d22869a344f 100644 --- a/salt/modules/freebsdkmod.py +++ b/salt/modules/freebsdkmod.py @@ -23,7 +23,7 @@ def _new_mods(pre_mods, post_mods): pre.add(mod['module']) for mod in post_mods: post.add(mod['module']) - return list(post.difference(pre)) + return list(post - pre) def _rm_mods(pre_mods, post_mods): @@ -37,7 +37,7 @@ def _rm_mods(pre_mods, post_mods): pre.add(mod['module']) for mod in post_mods: post.add(mod['module']) - return list(pre.difference(post)) + return list(pre - post) def available(): diff --git a/salt/modules/freebsdservice.py b/salt/modules/freebsdservice.py index 7073b06c7220..48ab4e9b5fd2 100644 --- a/salt/modules/freebsdservice.py +++ b/salt/modules/freebsdservice.py @@ -43,7 +43,7 @@ def get_disabled(): ''' en_ = get_enabled() all_ = get_all() - return sorted(set(all_).difference(set(en_))) + return sorted(set(all_) - set(en_)) def get_all(): @@ -57,8 +57,7 @@ def get_all(): for srv in ret: if srv.isupper(): rm_.add(srv) - ret.difference(rm_) - return sorted(ret) + return sorted(ret - rm_) def start(name): diff --git a/salt/modules/kmod.py b/salt/modules/kmod.py index a6dfd77d46eb..00018d3d924d 100644 --- a/salt/modules/kmod.py +++ b/salt/modules/kmod.py @@ -23,7 +23,7 @@ def _new_mods(pre_mods, post_mods): pre.add(mod['module']) for mod in post_mods: post.add(mod['module']) - return list(post.difference(pre)) + return list(post - pre) def _rm_mods(pre_mods, post_mods): @@ -37,7 +37,7 @@ def _rm_mods(pre_mods, post_mods): pre.add(mod['module']) for mod in post_mods: post.add(mod['module']) - return list(pre.difference(post)) + return list(pre - post) def available(): diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index 2b62807b7623..375eb28c4a6b 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -50,7 +50,7 @@ def _sync(form, env): open(os.path.join(__opts__['cachedir'], 'module_refresh'), 'w+').write('') if __opts__.get('clean_dynamic_modules', True): current = set(os.listdir(mod_dir)) - for fn_ in current.difference(remote): + for fn_ in current - remote: full = os.path.join(mod_dir, fn_) if os.path.isfile(full): os.remove(full) From 099470ab37568706883ada75d94114a022c8433d Mon Sep 17 00:00:00 2001 From: Eivind Uggedal Date: Mon, 20 Feb 2012 00:07:38 +0100 Subject: [PATCH 031/598] Terminate master and minion processes after tests are finished. --- tests/runtests.py | 8 ++++---- tests/saltunittest.py | 21 ++++++++++++++------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/tests/runtests.py b/tests/runtests.py index deea0875224e..3e2f7d878014 100644 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -10,10 +10,10 @@ TEST_DIR = os.path.dirname(os.path.normpath(os.path.abspath(__file__))) def main(): - saltunittest.TestDaemon() - loader = saltunittest.TestLoader() - tests = loader.discover(os.path.join(TEST_DIR, 'modules'), '*.py') - saltunittest.TextTestRunner(verbosity=1).run(tests) + with saltunittest.TestDaemon(): + loader = saltunittest.TestLoader() + tests = loader.discover(os.path.join(TEST_DIR, 'modules'), '*.py') + saltunittest.TextTestRunner(verbosity=1).run(tests) if __name__ == "__main__": diff --git a/tests/saltunittest.py b/tests/saltunittest.py index b4f7bd22c4fa..a520f8536052 100644 --- a/tests/saltunittest.py +++ b/tests/saltunittest.py @@ -45,7 +45,8 @@ class TestDaemon(object): ''' Set up the master and minion daemons, and run related cases ''' - def __init__(self): + + def __enter__(self): ''' Start a master and minion ''' @@ -59,18 +60,24 @@ def __init__(self): minion_opts['extension_modules'], master_opts['sock_dir'], ]) - # Start the master + master = salt.master.Master(master_opts) - multiprocessing.Process(target=master.start).start() - # Start the minion + self.master_process = multiprocessing.Process(target=master.start) + self.master_process.start() + minion = salt.minion.Minion(minion_opts) - multiprocessing.Process(target=minion.tune_in).start() + self.minion_process = multiprocessing.Process(target=minion.tune_in) + self.minion_process.start() + + return self + - def tearDown(self): + def __exit__(self, type, value, traceback): ''' Kill the minion and master processes ''' - pass + self.minion_process.terminate() + self.master_process.terminate() class ModuleCase(TestCase): From 361d1f9eb181a3d9a93966a4883c5519ce85e779 Mon Sep 17 00:00:00 2001 From: Christer Edwards Date: Sun, 19 Feb 2012 22:46:10 -0700 Subject: [PATCH 032/598] removed instructions re: path changes the Makefile has been patched so those manual changes are no longer needed. Also updated the tested versions. --- doc/topics/tutorials/freebsd.rst | 40 +++----------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/doc/topics/tutorials/freebsd.rst b/doc/topics/tutorials/freebsd.rst index 067509a5bb4a..6eb02ecf46cf 100644 --- a/doc/topics/tutorials/freebsd.rst +++ b/doc/topics/tutorials/freebsd.rst @@ -4,7 +4,7 @@ Introduction ============ Salt was added to the FreeBSD ports tree Dec 26th, 2011 by Christer Edwards -. It has been tested on FreeBSD 8.2 and 9.0RC +. It has been tested on FreeBSD 8.2 and 9.0 releases. Salt is dependent on the following additional ports. These will be installed as @@ -13,6 +13,7 @@ dependencies of the ``sysutils/salt`` port:: /devel/py-yaml /devel/py-pyzmq /devel/py-Jinja2 + /devel/py-msgpack /security/py-pycrypto /security/py-m2crypto @@ -70,30 +71,6 @@ By default the Salt master listens on ports 4505 and 4506 on all interfaces - #interface: 0.0.0.0 + interface: 10.0.0.1 -**pki_dir** - -Salt is primarily developed on Linux, and as such carries some Linux-isms in -its development and configuration. These are all very easily remedied to more -seamlessly fit into FreeBSD. One such configuration option is the *pki_dir:* -directive. To ensure all of Salts files end up where you expect, you'll likely -want to update this line as seen here:: - - - #pki_dir: /etc/salt/pki - + pki_dir: /usr/local/etc/salt/pki - -**file_roots** - -Finally, if you plan on using Salts state-management features, you'll want to -update the *file_roots:* directive. This directive defines the location of the -state files. I suggest updating this directive as seen here:: - - - #file_roots: - - # base: - - # - /srv/salt - + file_roots: - + base: - + - /usr/local/etc/salt/states - **rc.conf** Last but not least you'll need to activate the Salt Master in your rc.conf @@ -139,17 +116,6 @@ Simply update the master directive to the IP or hostname of your Salt Master. Save your changes and you're ready to start your Salt Minion. Advanced configuration options are covered in another chapter. -**pki_dir** - -Salt is primarily developed on Linux, and as such carries some Linux-isms in -its development and configuration. These are all very easily remedied to more -seamlessly fit into FreeBSD. One such configuration option is the *pki_dir:* -directive. To ensure all of Salts files end up where you expect, you'll likely -want to update this line as seen here:: - - - #pki_dir: /etc/salt/pki - + pki_dir: /usr/local/etc/salt/pki - **rc.conf** Before you're able to start the Salt Minion you'll need to update your rc.conf @@ -230,7 +196,7 @@ minions. If all of your minions are properly communicating with your Master, you should "True" responses from each of them. See the example below to send the ``test.ping`` remote command:: - [root@avon ~]# salt '*' test.ping + [root@master ~]# salt '*' test.ping {'avon': True} .. _where_do_i_go_from_here: From 2b7e71c7d8d7d8db6508c7c1197226074e946fb2 Mon Sep 17 00:00:00 2001 From: Robert James Hernandez Date: Sun, 19 Feb 2012 23:39:25 -0800 Subject: [PATCH 033/598] Fixed install method to accept the non-keyworded argument list along with the keyworded arguement list. This allowed me to run .sls for installing packages on minions. --- salt/modules/freebsdpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/freebsdpkg.py b/salt/modules/freebsdpkg.py index b3bfe1a3d9e5..63e0a4cb875d 100644 --- a/salt/modules/freebsdpkg.py +++ b/salt/modules/freebsdpkg.py @@ -85,7 +85,7 @@ def refresh_db(): __salt__['cmd.run']('portsnap update') -def install(name, **kwargs): +def install(name, *args, **kwargs): ''' Install the passed package From ab7ace6aa5cbf76314038e62d3c612392b8b76f8 Mon Sep 17 00:00:00 2001 From: Eivind Uggedal Date: Mon, 20 Feb 2012 13:18:13 +0100 Subject: [PATCH 034/598] Split out integration tests from pure unit tests. Integration tests requiring a salt master and daemon were seperated from pure unit tests. For now both are run with runtest.py. In the future it could take arguments for which type of tests to run. --- tests/integration/__init__.py | 94 +++++++++++++++++++ tests/{ => integration}/files/conf/master | 0 tests/{ => integration}/files/conf/minion | 2 +- tests/{ => integration}/files/hosts | 0 tests/{ => integration}/modules/files/hosts | 0 tests/{ => integration}/modules/grains.py | 6 +- tests/{ => integration}/modules/hosts.py | 8 +- tests/{ => integration}/modules/test.py | 6 +- tests/{ => integration}/tmp/_README | 0 tests/runtests.py | 15 ++- tests/saltunittest.py | 90 +----------------- tests/{ => unit}/simple.py | 0 tests/{ => unit}/templates/__init__.py | 0 .../templates/files/test/hello_import | 0 .../templates/files/test/hello_include | 0 .../templates/files/test/hello_simple | 0 tests/{ => unit}/templates/files/test/macro | 0 tests/{ => unit}/templates/jinja.py | 0 18 files changed, 116 insertions(+), 105 deletions(-) create mode 100644 tests/integration/__init__.py rename tests/{ => integration}/files/conf/master (100%) rename tests/{ => integration}/files/conf/minion (89%) rename tests/{ => integration}/files/hosts (100%) rename tests/{ => integration}/modules/files/hosts (100%) rename tests/{ => integration}/modules/grains.py (93%) rename tests/{ => integration}/modules/hosts.py (95%) rename tests/{ => integration}/modules/test.py (94%) rename tests/{ => integration}/tmp/_README (100%) rename tests/{ => unit}/simple.py (100%) rename tests/{ => unit}/templates/__init__.py (100%) rename tests/{ => unit}/templates/files/test/hello_import (100%) rename tests/{ => unit}/templates/files/test/hello_include (100%) rename tests/{ => unit}/templates/files/test/hello_simple (100%) rename tests/{ => unit}/templates/files/test/macro (100%) rename tests/{ => unit}/templates/jinja.py (100%) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 000000000000..6a782baf9c75 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,94 @@ +import multiprocessing +import os + +import salt +import salt.config +import salt.master +import salt.minion + +from saltunittest import TestCase + +INTEGRATION_TEST_DIR = os.path.dirname(os.path.normpath(os.path.abspath(__file__))) + +TMP = os.path.join(INTEGRATION_TEST_DIR, 'tmp') +FILES = os.path.join(INTEGRATION_TEST_DIR, 'files') + +class TestDaemon(object): + ''' + Set up the master and minion daemons, and run related cases + ''' + + def __enter__(self): + ''' + Start a master and minion + ''' + self.master_opts = salt.config.master_config(os.path.join(INTEGRATION_TEST_DIR, 'files/conf/master')) + self.minion_opts = salt.config.minion_config(os.path.join(INTEGRATION_TEST_DIR, 'files/conf/minion')) + salt.verify_env([os.path.join(self.master_opts['pki_dir'], 'minions'), + os.path.join(self.master_opts['pki_dir'], 'minions_pre'), + os.path.join(self.master_opts['pki_dir'], 'minions_rejected'), + os.path.join(self.master_opts['cachedir'], 'jobs'), + os.path.dirname(self.master_opts['log_file']), + self.minion_opts['extension_modules'], + self.master_opts['sock_dir'], + ]) + + master = salt.master.Master(self.master_opts) + self.master_process = multiprocessing.Process(target=master.start) + self.master_process.start() + + minion = salt.minion.Minion(self.minion_opts) + self.minion_process = multiprocessing.Process(target=minion.tune_in) + self.minion_process.start() + + return self + + + def __exit__(self, type, value, traceback): + ''' + Kill the minion and master processes + ''' + self.minion_process.terminate() + self.master_process.terminate() + + + +class ModuleCase(TestCase): + ''' + Execute a module function + ''' + def setUp(self): + ''' + Generate the tools to test a module + ''' + self.client = salt.client.LocalClient(os.path.join(INTEGRATION_TEST_DIR, 'files/conf/master')) + + def run_function(self, function, arg=()): + ''' + Run a single salt function and condition the return down to match the + behavior of the raw function call + ''' + orig = self.client.cmd('minion', function, arg) + return orig['minion'] + + def minion_opts(self): + ''' + Return the options used for the minion + ''' + return salt.config.minion_config( + os.path.join( + INTEGRATION_TEST_DIR, + 'files/conf/minion' + ) + ) + + def master_opts(self): + ''' + Return the options used for the minion + ''' + return salt.config.minion_config( + os.path.join( + INTEGRATION_TEST_DIR, + 'files/conf/master' + ) + ) diff --git a/tests/files/conf/master b/tests/integration/files/conf/master similarity index 100% rename from tests/files/conf/master rename to tests/integration/files/conf/master diff --git a/tests/files/conf/minion b/tests/integration/files/conf/minion similarity index 89% rename from tests/files/conf/minion rename to tests/integration/files/conf/minion index 206a238d904b..a3a745216cf6 100644 --- a/tests/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -11,7 +11,7 @@ log_file: /tmp/salttest/minion # module extension test.foo: baz -hosts.file: tmp/hosts +hosts.file: integration/tmp/hosts # Grains addons grains: diff --git a/tests/files/hosts b/tests/integration/files/hosts similarity index 100% rename from tests/files/hosts rename to tests/integration/files/hosts diff --git a/tests/modules/files/hosts b/tests/integration/modules/files/hosts similarity index 100% rename from tests/modules/files/hosts rename to tests/integration/modules/files/hosts diff --git a/tests/modules/grains.py b/tests/integration/modules/grains.py similarity index 93% rename from tests/modules/grains.py rename to tests/integration/modules/grains.py index 1a43e43b241f..a219644a019a 100644 --- a/tests/modules/grains.py +++ b/tests/integration/modules/grains.py @@ -2,11 +2,9 @@ Test the grains module ''' -# Import python libs -# Import Salt libs -import saltunittest +import integration -class TestModulesGrains(saltunittest.ModuleCase): +class TestModulesGrains(integration.ModuleCase): ''' Test the grains module ''' diff --git a/tests/modules/hosts.py b/tests/integration/modules/hosts.py similarity index 95% rename from tests/modules/hosts.py rename to tests/integration/modules/hosts.py index b64fa9226fc6..472473660d00 100644 --- a/tests/modules/hosts.py +++ b/tests/integration/modules/hosts.py @@ -6,11 +6,11 @@ import shutil # Import Salt libs -import saltunittest +import integration -HFN = os.path.join(saltunittest.TMP, 'hosts') +HFN = os.path.join(integration.TMP, 'hosts') -class HostsModuleTest(saltunittest.ModuleCase): +class HostsModuleTest(integration.ModuleCase): ''' Test the hosts module ''' @@ -18,7 +18,7 @@ def __clean_hosts(self): ''' Clean out the hosts file ''' - shutil.copyfile(os.path.join(saltunittest.FILES, 'hosts'), HFN) + shutil.copyfile(os.path.join(integration.FILES, 'hosts'), HFN) def __clear_hosts(self): ''' diff --git a/tests/modules/test.py b/tests/integration/modules/test.py similarity index 94% rename from tests/modules/test.py rename to tests/integration/modules/test.py index 0f0d0fa701c6..f9c0856a81ec 100644 --- a/tests/modules/test.py +++ b/tests/integration/modules/test.py @@ -2,9 +2,9 @@ import os # Import salt libs -import saltunittest +import integration -class TestModuleTest(saltunittest.ModuleCase): +class TestModuleTest(integration.ModuleCase): ''' Validate the test module ''' @@ -40,7 +40,7 @@ def test_get_opts(self): import salt.config opts = salt.config.minion_config( os.path.join( - saltunittest.TEST_DIR, + integration.INTEGRATION_TEST_DIR, 'files/conf/minion' ) ) diff --git a/tests/tmp/_README b/tests/integration/tmp/_README similarity index 100% rename from tests/tmp/_README rename to tests/integration/tmp/_README diff --git a/tests/runtests.py b/tests/runtests.py index 3e2f7d878014..ea49354aad07 100644 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -6,15 +6,22 @@ import os # Import salt libs import saltunittest +from integration import TestDaemon TEST_DIR = os.path.dirname(os.path.normpath(os.path.abspath(__file__))) -def main(): - with saltunittest.TestDaemon(): +def run_integration_tests(): + with TestDaemon(): loader = saltunittest.TestLoader() - tests = loader.discover(os.path.join(TEST_DIR, 'modules'), '*.py') + tests = loader.discover(os.path.join(TEST_DIR, 'integration', 'modules'), '*.py') saltunittest.TextTestRunner(verbosity=1).run(tests) +def run_unit_tests(): + loader = saltunittest.TestLoader() + tests = loader.discover(os.path.join(TEST_DIR, 'unit', 'templates'), '*.py') + saltunittest.TextTestRunner(verbosity=1).run(tests) + if __name__ == "__main__": - main() + run_integration_tests() + run_unit_tests() diff --git a/tests/saltunittest.py b/tests/saltunittest.py index a520f8536052..9f5a45af891a 100644 --- a/tests/saltunittest.py +++ b/tests/saltunittest.py @@ -8,7 +8,6 @@ """ # Import python libs -import multiprocessing import os import sys @@ -25,97 +24,10 @@ from unittest import TestLoader, TextTestRunner,\ TestCase, expectedFailure, \ TestSuite + # Set up paths TEST_DIR = os.path.dirname(os.path.normpath(os.path.abspath(__file__))) SALT_LIBS = os.path.dirname(TEST_DIR) -TMP = os.path.join(TEST_DIR, 'tmp') -FILES = os.path.join(TEST_DIR, 'files') sys.path.insert(0, TEST_DIR) sys.path.insert(0, SALT_LIBS) - -# Import salt libs -import salt -import salt.config -import salt.master -import salt.minion - - -class TestDaemon(object): - ''' - Set up the master and minion daemons, and run related cases - ''' - - def __enter__(self): - ''' - Start a master and minion - ''' - master_opts = salt.config.master_config(os.path.join(TEST_DIR, 'files/conf/master')) - minion_opts = salt.config.minion_config(os.path.join(TEST_DIR, 'files/conf/minion')) - salt.verify_env([os.path.join(master_opts['pki_dir'], 'minions'), - os.path.join(master_opts['pki_dir'], 'minions_pre'), - os.path.join(master_opts['pki_dir'], 'minions_rejected'), - os.path.join(master_opts['cachedir'], 'jobs'), - os.path.dirname(master_opts['log_file']), - minion_opts['extension_modules'], - master_opts['sock_dir'], - ]) - - master = salt.master.Master(master_opts) - self.master_process = multiprocessing.Process(target=master.start) - self.master_process.start() - - minion = salt.minion.Minion(minion_opts) - self.minion_process = multiprocessing.Process(target=minion.tune_in) - self.minion_process.start() - - return self - - - def __exit__(self, type, value, traceback): - ''' - Kill the minion and master processes - ''' - self.minion_process.terminate() - self.master_process.terminate() - - -class ModuleCase(TestCase): - ''' - Execute a module function - ''' - def setUp(self): - ''' - Generate the tools to test a module - ''' - self.client = salt.client.LocalClient('files/conf/master') - - def run_function(self, function, arg=()): - ''' - Run a single salt function and condition the return down to match the - behavior of the raw function call - ''' - orig = self.client.cmd('minion', function, arg) - return orig['minion'] - - def minion_opts(self): - ''' - Return the options used for the minion - ''' - return salt.config.minion_config( - os.path.join( - TEST_DIR, - 'files/conf/minion' - ) - ) - - def master_opts(self): - ''' - Return the options used for the minion - ''' - return salt.config.minion_config( - os.path.join( - TEST_DIR, - 'files/conf/master' - ) - ) diff --git a/tests/simple.py b/tests/unit/simple.py similarity index 100% rename from tests/simple.py rename to tests/unit/simple.py diff --git a/tests/templates/__init__.py b/tests/unit/templates/__init__.py similarity index 100% rename from tests/templates/__init__.py rename to tests/unit/templates/__init__.py diff --git a/tests/templates/files/test/hello_import b/tests/unit/templates/files/test/hello_import similarity index 100% rename from tests/templates/files/test/hello_import rename to tests/unit/templates/files/test/hello_import diff --git a/tests/templates/files/test/hello_include b/tests/unit/templates/files/test/hello_include similarity index 100% rename from tests/templates/files/test/hello_include rename to tests/unit/templates/files/test/hello_include diff --git a/tests/templates/files/test/hello_simple b/tests/unit/templates/files/test/hello_simple similarity index 100% rename from tests/templates/files/test/hello_simple rename to tests/unit/templates/files/test/hello_simple diff --git a/tests/templates/files/test/macro b/tests/unit/templates/files/test/macro similarity index 100% rename from tests/templates/files/test/macro rename to tests/unit/templates/files/test/macro diff --git a/tests/templates/jinja.py b/tests/unit/templates/jinja.py similarity index 100% rename from tests/templates/jinja.py rename to tests/unit/templates/jinja.py From 35ea9ed02dbc8f0bd1572e7970f218f86b7316c2 Mon Sep 17 00:00:00 2001 From: Eivind Uggedal Date: Mon, 20 Feb 2012 13:22:02 +0100 Subject: [PATCH 035/598] Kill all child processes of the test master daemon after each test run. --- tests/integration/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 6a782baf9c75..feac49f35e6c 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,5 +1,6 @@ import multiprocessing import os +import signal import salt import salt.config @@ -49,9 +50,19 @@ def __exit__(self, type, value, traceback): Kill the minion and master processes ''' self.minion_process.terminate() + self.stop_master_processes() self.master_process.terminate() + def stop_master_processes(self): + with open(self.master_opts['pidfile']) as pidfile: + for pid in pidfile.readlines(): + if len(pid.strip()): + try: + os.kill(int(pid.strip()), signal.SIGTERM) + except OSError: + pass + class ModuleCase(TestCase): ''' From fce9bd3e13d304e440b3a165c001559e12fc6ca0 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 08:40:52 -0700 Subject: [PATCH 036/598] clean up leftover line in salt spec --- pkg/rpm/salt.spec | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/rpm/salt.spec b/pkg/rpm/salt.spec index 71a647709a2a..7caac145a67c 100644 --- a/pkg/rpm/salt.spec +++ b/pkg/rpm/salt.spec @@ -93,7 +93,6 @@ Requires: salt = %{version}-%{release} %description -n salt-master The Salt master is the central server to which all minions connect. -Summary: %package -n salt-minion Summary: Client component for salt, a parallel remote execution system From 2bf942111c02f6c80959c001e0fbce05b81a483d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 11:55:53 -0700 Subject: [PATCH 037/598] invalidate extention functions in the state compiler I have decided that adding compiler hook capabilities in general to the state compiler and state python modules. This commit adds issolation of the compiler hooks from being called directly from the sls files (yes, that would have caused a trace) --- salt/state.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/state.py b/salt/state.py index 57a5eba6e7fa..ced262aabf90 100644 --- a/salt/state.py +++ b/salt/state.py @@ -24,6 +24,9 @@ log = logging.getLogger(__name__) +RESTRICTED_FUNCS = ('mod_init', 'watcher') + + def _gen_tag(low): ''' Generate the running dict tag string from the low data structure @@ -192,6 +195,13 @@ def verify_data(self, data): errors.append('Missing "name" data') if errors: return errors + if data['fun'] in RESTRICTED_FUNCS: + errors.append( + 'State {0} in sls {1} uses an invalid function {2}'.format( + data['state'], + data['__sls__'], + data['fun']) + ) full = data['state'] + '.' + data['fun'] if full not in self.states: if '__sls__' in data: From 97ef596a17123f7892009de0363fab95f06beb85 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 13:55:07 -0700 Subject: [PATCH 038/598] Scientific Linux needs to use rh_service module. --- salt/modules/rh_service.py | 1 + salt/modules/service.py | 1 + 2 files changed, 2 insertions(+) diff --git a/salt/modules/rh_service.py b/salt/modules/rh_service.py index be00c38c49e5..39ceb91e827f 100644 --- a/salt/modules/rh_service.py +++ b/salt/modules/rh_service.py @@ -14,6 +14,7 @@ def __virtual__(): enable = [ 'RedHat', 'CentOS', + 'Scientific', 'Fedora', ] if __grains__['os'] in enable: diff --git a/salt/modules/service.py b/salt/modules/service.py index a29f1b989b99..3fd8f8807d29 100644 --- a/salt/modules/service.py +++ b/salt/modules/service.py @@ -23,6 +23,7 @@ def __virtual__(): disable = [ 'RedHat', 'CentOS', + 'Scientific', 'Fedora', 'Gentoo', 'FreeBSD', From 360aa9792379b5bca9792bc0f311e2105e49cf9f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 15:02:51 -0700 Subject: [PATCH 039/598] Add tag for what order the states executed in for the state return --- salt/state.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/salt/state.py b/salt/state.py index ced262aabf90..41bed09bcd1f 100644 --- a/salt/state.py +++ b/salt/state.py @@ -16,7 +16,7 @@ import logging import os import tempfile -from collections import defaultdict +import collections import salt.loader import salt.minion @@ -108,6 +108,7 @@ def __init__(self, opts): self.opts = opts self.load_modules() self.mod_init = set() + self.__run_num = 0 def _mod_init(self, low): ''' @@ -553,6 +554,8 @@ def call(self, data): ) cdata = self.format_call(data) ret = self.states[cdata['full']](*cdata['args']) + ret['__run_num__'] = self.__run_num + self.__run_num += 1 format_log(ret) self.module_refresh(data) return ret @@ -791,9 +794,9 @@ def get_tops(self): ''' Gather the top files ''' - tops = defaultdict(list) - include = defaultdict(list) - done = defaultdict(list) + tops = collections.defaultdict(list) + include = collections.defaultdict(list) + done = collections.defaultdict(list) # Gather initial top files if self.opts['environment']: tops[self.opts['environment']] = [ @@ -855,7 +858,7 @@ def merge_tops(self, tops): ''' Cleanly merge the top files ''' - top = defaultdict(dict) + top = collections.defaultdict(dict) for sourceenv, ctops in tops.items(): for ctop in ctops: for env, targets in ctop.items(): From 3fd23a47afb89811eff46436f4f6b2468e6bfc5e Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 15:04:36 -0700 Subject: [PATCH 040/598] print out states after the order of execution --- salt/output.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/output.py b/salt/output.py index 8343d7792511..3419a11e733a 100644 --- a/salt/output.py +++ b/salt/output.py @@ -64,7 +64,8 @@ def __call__(self, data, **kwargs): .format(hcolor, err, colors))) if isinstance(data[host], dict): # Everything rendered as it should display the output - for tname, ret in data[host].items(): + for tname in sorted(data[host], key=lambda k: data[host][k]['__run_num__']): + ret = data[host][tname] tcolor = colors['GREEN'] if ret['changes']: tcolor = colors['CYAN'] From 038dea31d45cf741d44013bef68b155f8e253482 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 15:38:03 -0700 Subject: [PATCH 041/598] change watcher function to mod_watch I am moving all of the state module modifiers (watcher, mod_init) to function under a unified "mod" interface, watcher is being moved to mod_watch --- salt/state.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/salt/state.py b/salt/state.py index 41bed09bcd1f..587786c4bb0d 100644 --- a/salt/state.py +++ b/salt/state.py @@ -24,8 +24,6 @@ log = logging.getLogger(__name__) -RESTRICTED_FUNCS = ('mod_init', 'watcher') - def _gen_tag(low): ''' @@ -196,7 +194,7 @@ def verify_data(self, data): errors.append('Missing "name" data') if errors: return errors - if data['fun'] in RESTRICTED_FUNCS: + if data['fun'].startswith('mod_'): errors.append( 'State {0} in sls {1} uses an invalid function {2}'.format( data['state'], @@ -237,9 +235,9 @@ def verify_data(self, data): if 'require' in data: reqdec = 'require' if 'watch' in data: - # Check to see if the service has a watcher function, if it does + # Check to see if the service has a mod_watch function, if it does # not, then just require - if not '{0}.watcher'.format(data['state']) in self.states: + if not '{0}.mod_watch'.format(data['state']) in self.states: data['require'] = data.pop('watch') reqdec = 'require' else: @@ -696,7 +694,7 @@ def call_chunk(self, low, running, chunks): elif status == 'change': ret = self.call(low) if not ret['changes']: - low['fun'] = 'watcher' + low['fun'] = 'mod_watch' ret = self.call(low) running[tag] = ret else: From ec8454a3b2fb69ab3192e89934dbf68f8b5aa7b6 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 15:41:05 -0700 Subject: [PATCH 042/598] change all watcher functions to mod_watch --- salt/states/cmd.py | 2 +- salt/states/service.py | 42 +++++++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/salt/states/cmd.py b/salt/states/cmd.py index 652153801dcc..68bd9b5602b1 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -149,4 +149,4 @@ def run(name, os.setegid(pgid) return ret -watcher = run +mod_watch = run diff --git a/salt/states/service.py b/salt/states/service.py index c36a2b1289b9..9d4c3816d801 100644 --- a/salt/states/service.py +++ b/salt/states/service.py @@ -291,7 +291,27 @@ def dead(name, enable=None, sig=None): return ret -def watcher(name, sig=None): +def enabled(name): + ''' + Verify that the service is enabled on boot + + name + The name of the init or rc script used to manage the service + ''' + return _enable(name, None) + + +def disabled(name): + ''' + Verify that the service is disabled on boot + + name + The name of the init or rc script used to manage the service + ''' + return _disable(name, None) + + +def mod_watch(name, sig=None): ''' The service watcher, called to invoke the watch command. @@ -312,23 +332,3 @@ def watcher(name, sig=None): 'changes': {}, 'result': True, 'comment': 'Service {0} started'.format(name)} - - -def enabled(name): - ''' - Verify that the service is enabled on boot - - name - The name of the init or rc script used to manage the service - ''' - return _enable(name, None) - - -def disabled(name): - ''' - Verify that the service is disabled on boot - - name - The name of the init or rc script used to manage the service - ''' - return _disable(name, None) From efaef20bd3a8a1615bbdc45ebf05928b3aa4ac52 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 15:42:17 -0700 Subject: [PATCH 043/598] move prve funcs to the top of modules --- salt/states/pkg.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 421510a88298..e485a4913c95 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -18,6 +18,14 @@ logger = logging.getLogger(__name__) + +def __gen_rtag(): + ''' + Return the location of the refresh tag + ''' + return os.path.join(__opts__['cachedir'], 'pkg_refresh') + + def installed(name, version=None, refresh=False, repo='', skip_verify=False): ''' Verify that the package is installed, and only that it is installed. This @@ -206,9 +214,3 @@ def mod_init(low): open(rtag, 'w+').write('') return True return False - -def __gen_rtag(): - ''' - Return the location of the refresh tag - ''' - return os.path.join(__opts__['cachedir'], 'pkg_refresh') From eb97e55252d7aa04dd0ea48cb7f954842e27c89b Mon Sep 17 00:00:00 2001 From: Christer Edwards Date: Mon, 20 Feb 2012 16:37:12 -0700 Subject: [PATCH 044/598] created archlinux installation tutorial --- doc/topics/tutorials/archlinux.rst | 258 +++++++++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 doc/topics/tutorials/archlinux.rst diff --git a/doc/topics/tutorials/archlinux.rst b/doc/topics/tutorials/archlinux.rst new file mode 100644 index 000000000000..7af629fccd0e --- /dev/null +++ b/doc/topics/tutorials/archlinux.rst @@ -0,0 +1,258 @@ +.. _introduction: + +Introduction +============ + +Salt has primarily been developed on Arch Linux, meaning it is known to work +very well on that distribution. The lead developer, Thomas S. Hatch (thatch45) has +been a TU (Trusted User) for the Arch Linux distribution, and has written a +number of Arch-specific tools in the past. + +Salt, while not Arch-specific, is packaged for and works well on Arch Linux. + +.. _installation: + +Installation +============ + +Salt is currently available via the Arch User Repository (AUR). There are +currently stable and -git packages available. + +Stable Release +-------------- + +To install Salt stable releases from the Arch Linux AUR, use the commands:: + + wget https://aur.archlinux.org/packages/sa/salt/salt.tar.gz + tar xf salt.tar.gz + cd salt/ + makepkg -is + +A few of Salt's dependencies are currently only found within the AUR, so you'll +need to download and run ``makepkg -is`` on these as well. As a reference, Salt +currently relies on the following packages only available via the AUR: + +* https://aur.archlinux.org/packages/py/python2-msgpack/python2-msgpack.tar.gz +* https://aur.archlinux.org/packages/py/python2-psutil/python2-psutil.tar.gz + +.. note:: yaourt + + If you chose to use a tool such as Yaourt_ the dependencies will be + gathered and built for you automatically. + + The command to install salt using the yaourt tool is: + + .. code-block:: none + + yaourt salt + +.. _Yaourt: https://aur.archlinux.org/packages.php?ID=5863 + +Tracking develop +---------------- + +To install the bleeding edge version of Salt (**may include bugs!**), you can use +the -git package. Installing the -git package can be done using the commands:: + + wget https://aur.archlinux.org/packages/sa/salt-git/salt-git.tar.gz + tar xf salt-git.tar.gz + cd salt-git/ + makepkg -is + +A few of Salt's dependencies are currently only found within the AUR, so you'll +need to download and run ``makepkg -is`` on these as well. As a reference, Salt +currently relies on the following packages only available via the AUR: + +* https://aur.archlinux.org/packages/py/python2-msgpack/python2-msgpack.tar.gz +* https://aur.archlinux.org/packages/py/python2-psutil/python2-psutil.tar.gz + +.. note:: yaourt + + If you chose to use a tool such as Yaourt_ the dependencies will be + gathered and built for you automatically. + + The command to install salt using the yaourt tool is: + + .. code-block:: none + + yaourt salt-git + +.. _Yaourt: https://aur.archlinux.org/packages.php?ID=5863 + +.. _configuration: + +Configuration +============= + +In the sections below I'll outline configuration options for both the Salt +Master and Salt Minions. + +The Salt package installs two template configuration files, /etc/salt/master.template and +/etc/salt/minion.template. You'll need to copy these .template files into place and +make a few edits. First, copy them into place as seen here:: + + cp /etc/salt/master.template /etc/salt/master + cp /etc/salt/minion.template /etc/salt/minion + +Note: You'll only need to copy the config for the service you're going to run. + +Once you've copied the config into place you'll need to make changes specific +to your setup. Below I'll outline suggested configuration changes to the +Master, after which I'll outline configuring the Minion. + +.. _master_configuration: + +Master Configuration +==================== + +This section outlines configuration of a Salt Master, which is used to control +other machines known as "minions" (see "Minion Configuration" for instructions +on configuring a minion). This will outline IP configuration, and a few key +configuration paths. + +**Interface** + +By default the Salt master listens on ports 4505 and 4506 on all interfaces +(0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the +"interface" directive as seen here:: + + - #interface: 0.0.0.0 + + interface: 10.0.0.1 + +**rc.conf** + +Last but not least you'll need to activate the Salt Master in your rc.conf +file. Using your favorite editor, open /etc/rc.conf and add the salt-master:: + + -DAEMONS=(syslog-ng network crond) + +DAEMONS=(syslog-ng network crond @salt-master) + +Once you've completed all of these steps you're ready to start your Salt +Master. You should be able to start your Salt Master now using the command +seen here:: + + rc.d start salt-master + +If your Salt Master doesn't start successfully, go back through each step and +see if anything was missed. Salt doesn't take much configuration (part of its +beauty!), and errors are usually simple mistakes. + +.. _ minion_configuration: + +Minion Configuration +==================== + +Configuring a Salt Minion is surprisingly simple. Unless you have a real need +for customizing your minion configuration (which there are plenty of options if +you are so inclined!), there is one simple directive that needs to be updated. +That option is the location of the master. + +By default a Salt Minion will try to connect to the dns name "salt". If you +have the ability to update DNS records for your domain you might create an A or +CNAME record for "salt" that points to your Salt Master. If you are able to do +this you likely can do without any minion configuration at all. + +If you are not able to update DNS, you'll simply need to update one entry in +the configuration file. Using your favorite editor, open the minion +configuration file and update the "master" entry as seen here:: + + - #master: salt + + master: 10.0.0.1 + +Simply update the master directive to the IP or hostname of your Salt Master. +Save your changes and you're ready to start your Salt Minion. Advanced +configuration options are covered in another chapter. + +**rc.conf** + +Before you're able to start the Salt Minion you'll need to update your rc.conf +file. Using your favorite editor open /etc/rc.conf or /etc/rc.conf.local and +add this line:: + + -DAEMONS=(syslog-ng network crond) + +DAEMONS=(syslog-ng network crond @salt-minion) + +Once you've completed all of these steps you're ready to start your Salt +Minion. You should be able to start your Salt Minion now using the command +seen here:: + + rc.d start salt-minion + +If your Salt Minion doesn't start successfully, go back through each step and +see if anything was missed. Salt doesn't take much configuration (part of its +beauty!), and errors are usually simple mistakes. + +.. _tying_it_all_together: + +Tying It All Together +====================== + +If you've successfully completed each of the steps above you should have a +running Salt Master and a running Salt Minion. The Minion should be configured +to point to the Master. To verify that there is communication flowing between +the Minion and Master we'll run a few initial ``salt`` commands. These commands +will validate the Minions RSA encryption key, and then send a test command to +the Minion to ensure that commands and responses are flowing as expected. + +**Key Management** + +Salt uses AES encryption for all communication between the Master and the +Minion. This ensures that the commands you send to your Minions (your cloud) +can not be tampered with, and that communication between Master and Minion is +only done through trusted, accepted keys. + +Before you'll be able to do any remote execution or configuration management you'll +need to accept any pending keys on the Master. Run the ``salt-key`` command to +list the keys known to the Salt Master:: + + [root@master ~]# salt-key -L + Unaccepted Keys: + avon + bodie + bubbles + marlo + Accepted Keys: + +This example shows that the Salt Master is aware of four Minions, but none of +the keys have been accepted. To accept the keys and allow the Minions to be +controlled by the Master, again use the ``salt-key`` command:: + + [root@master ~]# salt-key -A + [root@master ~]# salt-key -L + Unaccepted Keys: + Accepted Keys: + avon + bodie + bubbles + marlo + +The ``salt-key`` command allows for signing keys individually or in bulk. The +example above, using ``-A`` bulk-accepts all pending keys. To accept keys +individually use the lowercase of the same option, ``-a keyname``. + +.. _sending_commands: + +Sending Commands +================ + +Everything should be set for you to begin remote management of your Minions. +Whether you have a few or a few-dozen, Salt can help you manage them easily! + +For final verification, send a test function from your Salt Master to your +minions. If all of your minions are properly communicating with your Master, +you should "True" responses from each of them. See the example below to send +the ``test.ping`` remote command:: + + [root@master ~]# salt '*' test.ping + {'avon': True} + +.. _where_do_i_go_from_here: + +Where Do I Go From Here +======================== + +Congratulations! You've successfully configured your first Salt Minions and are +able to send remote commands. I'm sure you're eager to learn more about what +Salt can do. Depending on the primary way you want to manage your machines you +may either want to visit the section regarding Salt States, or the section on +Modules. From 65b52df662bb89ae602bd0f9a7bf6b2fc89ad0ab Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 16:50:56 -0700 Subject: [PATCH 045/598] Add run_num to requisite error returns --- salt/state.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index 587786c4bb0d..523bc961ec49 100644 --- a/salt/state.py +++ b/salt/state.py @@ -670,7 +670,9 @@ def call_chunk(self, low, running, chunks): lreq) running[tag] = {'changes': {}, 'result': False, - 'comment': comment} + 'comment': comment, + '__run_num__': self.__run_num} + self.__run_num += 1 return running for chunk in reqs: # Check to see if the chunk has been run, only run it if @@ -690,7 +692,9 @@ def call_chunk(self, low, running, chunks): elif status == 'fail': running[tag] = {'changes': {}, 'result': False, - 'comment': 'One or more requisite failed'} + 'comment': 'One or more requisite failed', + '__run_num__': self.run_num} + self.__run_num += 1 elif status == 'change': ret = self.call(low) if not ret['changes']: From e47f498d7cce7e67811e0a78c69ae3cce6732016 Mon Sep 17 00:00:00 2001 From: Christer Edwards Date: Mon, 20 Feb 2012 18:29:28 -0700 Subject: [PATCH 046/598] created basic installation tutorial for fedora 16 --- doc/topics/tutorials/fedora.rst | 189 ++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 doc/topics/tutorials/fedora.rst diff --git a/doc/topics/tutorials/fedora.rst b/doc/topics/tutorials/fedora.rst new file mode 100644 index 000000000000..d38efc305a11 --- /dev/null +++ b/doc/topics/tutorials/fedora.rst @@ -0,0 +1,189 @@ +.. _introduction: + +Introduction +============ + +Beginning with version 0.9.4, Salt has been available in the primary Fedora +repositories and is available for installation using yum. Fedora will have more +up to date versions of Salt than other members of the Red Hat family, which +makes it a great place to help improve Salt! + +.. _installation: + +Installation +============ + +Salt can be installed using ``yum`` and is available in the standard Fedora +repositories. + +Stable Release +-------------- + +Salt is packaged separately for the minion and the master. You'll only need to +install the appropriate package for the role you need the machine to play. This +means you're going to want one master and a whole bunch of minions!:: + + yum install salt-master + yum install salt-minion + +.. _configuration: + +Configuration +============= + +In the sections below I'll outline configuration options for both the Salt +Master and Salt Minions. + +.. _master_configuration: + +Master Configuration +==================== + +This section outlines configuration of a Salt Master, which is used to control +other machines known as "minions" (see "Minion Configuration" for instructions +on configuring a minion). This will outline IP configuration, and a few key +configuration paths. + +**Interface** + +By default the Salt master listens on ports 4505 and 4506 on all interfaces +(0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the +"interface" directive as seen here:: + + - #interface: 0.0.0.0 + + interface: 10.0.0.1 + +**Enable the Master** + +You'll also likely want to activate the Salt Master in systemd, configuring the +Salt Master to start automatically at boot.:: + + systemctl enable salt-master.service + +Once you've completed all of these steps you're ready to start your Salt +Master. You should be able to start your Salt Master now using the command +seen here:: + + systemctl start salt-master.service + +If your Salt Master doesn't start successfully, go back through each step and +see if anything was missed. Salt doesn't take much configuration (part of its +beauty!), and errors are usually simple mistakes. + +.. _ minion_configuration: + +Minion Configuration +==================== + +Configuring a Salt Minion is surprisingly simple. Unless you have a real need +for customizing your minion configuration (which there are plenty of options if +you are so inclined!), there is one simple directive that needs to be updated. +That option is the location of the master. + +By default a Salt Minion will try to connect to the dns name "salt". If you +have the ability to update DNS records for your domain you might create an A or +CNAME record for "salt" that points to your Salt Master. If you are able to do +this you likely can do without any minion configuration at all. + +If you are not able to update DNS, you'll simply need to update one entry in +the configuration file. Using your favorite editor, open the minion +configuration file and update the "master" entry as seen here:: + + - #master: salt + + master: 10.0.0.1 + +Simply update the master directive to the IP or hostname of your Salt Master. +Save your changes and you're ready to start your Salt Minion. Advanced +configuration options are covered in another chapter. + +**Enable the Minion** + +You'll need to configure the minion to auto-start at boot. You can toggle +that option through systemd.:: + + systemctl enable salt-minion.service + +Once you've completed all of these steps you're ready to start your Salt +Minion. You should be able to start your Salt Minion now using the command +here:: + + systemctl start salt-minion.service + +If your Salt Minion doesn't start successfully, go back through each step and +see if anything was missed. Salt doesn't take much configuration (part of its +beauty!), and errors are usually simple mistakes. + +.. _tying_it_all_together: + +Tying It All Together +====================== + +If you've successfully completed each of the steps above you should have a +running Salt Master and a running Salt Minion. The Minion should be configured +to point to the Master. To verify that there is communication flowing between +the Minion and Master we'll run a few initial ``salt`` commands. These commands +will validate the Minions RSA encryption key, and then send a test command to +the Minion to ensure that commands and responses are flowing as expected. + +**Key Management** + +Salt uses AES encryption for all communication between the Master and the +Minion. This ensures that the commands you send to your Minions (your cloud) +can not be tampered with, and that communication between Master and Minion is +only done through trusted, accepted keys. + +Before you'll be able to do any remote execution or configuration management you'll +need to accept any pending keys on the Master. Run the ``salt-key`` command to +list the keys known to the Salt Master:: + + [root@master ~]# salt-key -L + Unaccepted Keys: + avon + bodie + bubbles + marlo + Accepted Keys: + +This example shows that the Salt Master is aware of four Minions, but none of +the keys have been accepted. To accept the keys and allow the Minions to be +controlled by the Master, again use the ``salt-key`` command:: + + [root@master ~]# salt-key -A + [root@master ~]# salt-key -L + Unaccepted Keys: + Accepted Keys: + avon + bodie + bubbles + marlo + +The ``salt-key`` command allows for signing keys individually or in bulk. The +example above, using ``-A`` bulk-accepts all pending keys. To accept keys +individually use the lowercase of the same option, ``-a keyname``. + +.. _sending_commands: + +Sending Commands +================ + +Everything should be set for you to begin remote management of your Minions. +Whether you have a few or a few-dozen, Salt can help you manage them easily! + +For final verification, send a test function from your Salt Master to your +minions. If all of your minions are properly communicating with your Master, +you should "True" responses from each of them. See the example below to send +the ``test.ping`` remote command:: + + [root@master ~]# salt '*' test.ping + {'avon': True} + +.. _where_do_i_go_from_here: + +Where Do I Go From Here +======================== + +Congratulations! You've successfully configured your first Salt Minions and are +able to send remote commands. I'm sure you're eager to learn more about what +Salt can do. Depending on the primary way you want to manage your machines you +may either want to visit the section regarding Salt States, or the section on +Modules. From a36b3d1858ad127bfe87b73c9faf737d7309bfb1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 19:51:25 -0700 Subject: [PATCH 047/598] Add file glob detection in requisites --- salt/state.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/salt/state.py b/salt/state.py index 523bc961ec49..e2f824905461 100644 --- a/salt/state.py +++ b/salt/state.py @@ -13,6 +13,7 @@ import copy import inspect +import fnmatch import logging import os import tempfile @@ -245,8 +246,8 @@ def verify_data(self, data): if reqdec: for req in data[reqdec]: if data['state'] == req.keys()[0]: - if data['name'] == req[req.keys()[0]] \ - or data['__id__'] == req[req.keys()[0]]: + if fnmatch.fnmatch(data['name'], req[req.keys()[0]]) \ + or fnmatch.fnmatch(data['__id__'], req[req.keys()[0]]): err = ('Recursive require detected in SLS {0} for' ' require {1} in ID {2}').format( data['__sls__'], @@ -604,8 +605,8 @@ def check_requisite(self, low, running, chunks): for req in low[r_state]: found = False for chunk in chunks: - if chunk['__id__'] == req[req.keys()[0]] or \ - chunk['name'] == req[req.keys()[0]]: + if fnmatch.fnmatch(chunk['__id__'], req[req.keys()[0]]) or \ + fnmatch.fnmatch(chunk['name'], req[req.keys()[0]]): if chunk['state'] == req.keys()[0]: found = True reqs[r_state].append(chunk) @@ -654,8 +655,9 @@ def call_chunk(self, low, running, chunks): for req in low[requisite]: found = False for chunk in chunks: - if chunk['name'] == req[req.keys()[0]] \ - or chunk['__id__'] == req[req.keys()[0]]: + if fnmatch.fnmatch(chunk['name'], req[req.keys()[0]]) \ + or fnmatch.fnmatch(chunk['__id__'], + req[req.keys()[0]]): if chunk['state'] == req.keys()[0]: reqs.append(chunk) found = True @@ -693,7 +695,7 @@ def call_chunk(self, low, running, chunks): running[tag] = {'changes': {}, 'result': False, 'comment': 'One or more requisite failed', - '__run_num__': self.run_num} + '__run_num__': self.__run_num} self.__run_num += 1 elif status == 'change': ret = self.call(low) From 9142f005b8919a934347da09840cf3922f0c980f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 22:50:25 -0700 Subject: [PATCH 048/598] regect replay return messages to the master --- salt/master.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/master.py b/salt/master.py index 57d2b45e00b4..a240b11d7b79 100644 --- a/salt/master.py +++ b/salt/master.py @@ -508,6 +508,15 @@ def _return(self, load): hn_dir = os.path.join(jid_dir, load['id']) if not os.path.isdir(hn_dir): os.makedirs(hn_dir) + # Otherwise the minion has already returned this jid and it should + # be dropped + else: + log.error( + ('An extra return was detected from minion {0}, please' + ' verify the minion, this could be a replay' + ' attack').format(load['id']) + ) + return False self.serial.dump(load['return'], open(os.path.join(hn_dir, 'return.p'), 'w+')) if 'out' in load: From 48d507b50517bc43f422836c303e6f210bc6e391 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 20 Feb 2012 22:51:08 -0700 Subject: [PATCH 049/598] reauth a message return if the AES key has changed --- salt/minion.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index d2f7a2586b54..653781352edd 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -361,7 +361,14 @@ def _return_pub(self, ret, ret_cmd='_return'): payload['load'] = self.crypticle.dumps(load) data = self.serial.dumps(payload) socket.send(data) - ret_val = socket.recv() + ret_val = self.serial.loads(socket.recv()) + if isinstance(ret_val, str) and not ret_val: + # The master AES key has changed, reauth + self.authenticate() + payload['load'] = self.crypticle.dumps(load) + data = self.serial.dumps(payload) + socket.send(data) + ret_val = self.serial.loads(socket.recv()) if self.opts['cache_jobs']: # Local job cache has been enabled fn_ = os.path.join( From f465dcff0b007be3fe51ded43b8c9eacedb636ae Mon Sep 17 00:00:00 2001 From: Robert James Hernandez Date: Tue, 21 Feb 2012 00:07:30 -0800 Subject: [PATCH 050/598] Added Rehashing for C shell after a package is installed --- salt/modules/freebsdpkg.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/salt/modules/freebsdpkg.py b/salt/modules/freebsdpkg.py index 63e0a4cb875d..6da1c5c80b22 100644 --- a/salt/modules/freebsdpkg.py +++ b/salt/modules/freebsdpkg.py @@ -115,6 +115,7 @@ def install(name, *args, **kwargs): # the package is freshly installed pkgs[npkg] = {'old': '', 'new': new[npkg]} + rehash() return pkgs @@ -148,6 +149,7 @@ def upgrade(): # the package is freshly installed pkgs[npkg] = {'old': '', 'new': new[npkg]} + rehash() return pkgs @@ -180,3 +182,17 @@ def purge(name): salt '*' pkg.purge ''' return remove(name) + +def rehash(): + ''' + Recomputes internal hash table for the PATH variable. + Use whenever a new command is created during the current + session. + + CLI Example:: + + salt '*' cmd.run 'rehash' + ''' + shell = __salt__['cmd.run']('echo $SHELL').split('/') + if shell[len(shell)-1] in ["csh","tcsh"]: + __salt__['cmd.run']('rehash') From ce7a7b3fe08e0cc35184c707bafef2e45f8c8c44 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 21 Feb 2012 10:21:02 -0700 Subject: [PATCH 051/598] Add some service docs --- salt/states/service.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/states/service.py b/salt/states/service.py index 9d4c3816d801..b3d1d61b34b2 100644 --- a/salt/states/service.py +++ b/salt/states/service.py @@ -293,7 +293,10 @@ def dead(name, enable=None, sig=None): def enabled(name): ''' - Verify that the service is enabled on boot + Verify that the service is enabled on boot, only use this state if you + don't want to manage the running process, remember that if you want to + enable a running service to use the enable: True option for the running + or dead function. name The name of the init or rc script used to manage the service @@ -303,7 +306,10 @@ def enabled(name): def disabled(name): ''' - Verify that the service is disabled on boot + Verify that the service is disabled on boot, only use this state if you + don't want to manage the running process, remember that if you want to + disable a service to use the enable: False option for the running or dead + function. name The name of the init or rc script used to manage the service From d7a0d13d1b46fd84908138845b61223873cae1f2 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 21 Feb 2012 12:11:20 -0700 Subject: [PATCH 052/598] Add detection of ID dec collisions in multiple sls files --- salt/state.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/state.py b/salt/state.py index e2f824905461..42018901d182 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1032,6 +1032,17 @@ def render_highstate(self, matches): mods = set() for sls in states: state, mods, err = self.render_state(sls, env, mods) + for id_ in state: + if id_ in highstate: + if highstate[id_] != state[id_]: + errors.append(('Detected conflicting IDs, SLS IDs' + ' need to be globally unique.\n The' + ' conflicting ID is "{0}" and is found in SLS' + ' "{1}" and SLS "{2}"').format( + id_, + highstate[id_]['__sls__'], + state[id_]['__sls__']) + ) if state: highstate.update(state) if err: From 1ed6b5e56ad765597b89c17ed55cd032e3437df9 Mon Sep 17 00:00:00 2001 From: Michael Lustfield Date: Tue, 21 Feb 2012 15:50:38 -0600 Subject: [PATCH 053/598] Simple typo in example usage of salt.modules.virt.freecpu() --- doc/man/salt.7 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/man/salt.7 b/doc/man/salt.7 index 0265b2ec632c..f306e5154786 100644 --- a/doc/man/salt.7 +++ b/doc/man/salt.7 @@ -7156,7 +7156,7 @@ CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.freemem +salt \(aq*\(aq virt.freecpu .ft P .fi .UNINDENT From ec3a7fd80b979eef24da80924949a2ea0fe9144a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 21 Feb 2012 14:54:44 -0700 Subject: [PATCH 054/598] fix doc in virt.freecpu --- salt/modules/virt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 12d708de4fa0..7bd4812dcebe 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -208,7 +208,7 @@ def freecpu(): CLI Example:: - salt '*' virt.freemem + salt '*' virt.freecpu ''' conn = __get_conn() cpus = conn.getInfo()[2] From 3df7714dc3b4ca251879c439b6b061dd383b9c4f Mon Sep 17 00:00:00 2001 From: Ben Hosmer Date: Tue, 21 Feb 2012 19:26:20 -0500 Subject: [PATCH 055/598] Adding documention to delete a minion key --- doc/man/salt-key.1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/man/salt-key.1 b/doc/man/salt-key.1 index 5bc9f56c82b7..1892e1f2fa88 100644 --- a/doc/man/salt-key.1 +++ b/doc/man/salt-key.1 @@ -73,6 +73,10 @@ Reject the named minion public key. .TP .B \-R, \-\-reject\-all Rejects all pending public keys. +.INDENT 0.0 +.TP +.B \-d, \-\-delete=DELETE +Delete the named minion public key for command execution. .UNINDENT .INDENT 0.0 .TP From 25981d24dba73bc18e19ba5d8120de3a72c16210 Mon Sep 17 00:00:00 2001 From: Ben Hosmer Date: Tue, 21 Feb 2012 19:36:22 -0500 Subject: [PATCH 056/598] Adding documentation for deleting a salt-key --- doc/ref/cli/salt-key.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/ref/cli/salt-key.rst b/doc/ref/cli/salt-key.rst index f9b62b27ea65..bab9463ddb68 100644 --- a/doc/ref/cli/salt-key.rst +++ b/doc/ref/cli/salt-key.rst @@ -47,6 +47,10 @@ Options Rejects all pending public keys. + option:: -d DELETE --delete=DELETE + + Delete the named minion key for command execution. + .. option:: -c CONFIG, --config=CONFIG The master configuration file needs to be read to determine where the salt From ae141cee54cf62fffdcb4e7260ed6adfb7898c81 Mon Sep 17 00:00:00 2001 From: James Gambill Date: Tue, 21 Feb 2012 18:56:33 -0800 Subject: [PATCH 057/598] - CommandExecutionError wasn't imported - testing github/pull works as advertised --- salt/states/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/cmd.py b/salt/states/cmd.py index 68bd9b5602b1..f3ecd7d53b80 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -36,7 +36,7 @@ import grp import os import pwd - +from salt.exceptions import CommandExecutionError def wait(name, onlyif=None, From 052ff2e58cfaf2e6c7dce46ee3bc1f885f2f512f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 21 Feb 2012 20:26:31 -0700 Subject: [PATCH 058/598] Add the capability to the loader to load a single named module This function allows a single module in the modulepath to be loaded, it is noteworthy that this functionality bypassed the __virtual__ function and allows for platform decisions to be bypassed entirely --- salt/loader.py | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/salt/loader.py b/salt/loader.py index e13927cc51e5..2234b8c79ecc 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -34,6 +34,23 @@ def minion_mods(opts): return load.apply_introspection(load.gen_functions()) +def raw_mod(opts, name, functions): + ''' + Returns a single module loaded raw and bypassing the __virtual__ function + ''' + extra_dirs = [ + os.path.join(opts['extension_modules'], + 'modules') + ] + if 'module_dirs' in opts: + extra_dirs.extend(opts['module_dirs']) + module_dirs = [ + os.path.join(salt_base_path, 'modules'), + ] + extra_dirs + load = Loader(module_dirs, opts, 'rawmodule') + return load.gen_module(name, functions) + + def returners(opts): ''' Returns the returner modules @@ -206,6 +223,79 @@ def call(self, fun, arg=list()): "modules.") return getattr(mod, fun[fun.rindex('.') + 1:])(*arg) + def gen_module(self, name, functions, pack=None): + ''' + Load a single module and pack it with the functions passed + ''' + full = '' + mod = None + for mod_dir in self.module_dirs: + if not os.path.isabs(mod_dir): + continue + if not os.path.isdir(mod_dir): + continue + fn_ = os.path.join(mod_dir, name) + for ext in ('.py', '.pyo', '.pyc', '.so'): + full_test = '{0}{1}'.format(fn_, ext) + if os.path.isfile(full_test): + full = full_test + if not full: + return None + try: + if full.endswith('.pyx') and self.opts['cython_enable']: + mod = pyximport.load_module(name, full, '/tmp') + else: + fn_, path, desc = imp.find_module(name, self.module_dirs) + mod = imp.load_module( + '{0}_{1}'.format(name, self.tag), + fn_, + path, + desc + ) + except ImportError as exc: + log.debug(('Failed to import module {0}: {1}').format(name, exc)) + return mod + except Exception as exc: + log.warning(('Failed to import module {0}, this is due most' + ' likely to a syntax error: {1}').format(name, exc)) + return mod + if hasattr(mod, '__opts__'): + mod.__opts__.update(self.opts) + else: + mod.__opts__ = self.opts + + mod.__grains__ = self.grains + + if pack: + if isinstance(pack, list): + for chunk in pack: + setattr(mod, chunk['name'], chunk['value']) + else: + setattr(mod, pack['name'], pack['value']) + + # Call a module's initialization method if it exists + if hasattr(mod, '__init__'): + if callable(mod.__init__): + try: + mod.__init__() + except TypeError: + pass + funcs = {} + for attr in dir(mod): + if attr.startswith('_'): + continue + if callable(getattr(mod, attr)): + func = getattr(mod, attr) + funcs[ + '{0}.{1}'.format( + mod.__name__[:mod.__name__.rindex('_')], + attr) + ] = func + self._apply_outputter(func, mod) + if not hasattr(mod, '__salt__'): + mod.__salt__ = functions + return funcs + def gen_functions(self, pack=None, virtual_enable=True): ''' Return a dict of functions found in the defined module_dirs From e26221340bbcd644d2ffdfb4e658e75ef479a85c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 21 Feb 2012 20:28:00 -0700 Subject: [PATCH 059/598] Add redirect global option to states The redirect global option allows for underlying modules to be redirected for a particular state. for instance the pkg module can be redirected to use a specific pkg backend module insted of the default platform specific module. --- salt/state.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index 42018901d182..504e8d54cb36 100644 --- a/salt/state.py +++ b/salt/state.py @@ -123,12 +123,27 @@ def _mod_init(self, low): return self.mod_init.add(low['state']) - def load_modules(self): + def load_modules(self, data=None): ''' Load the modules into the state ''' log.info('Loading fresh modules for state activity') self.functions = salt.loader.minion_mods(self.opts) + if isinstance(data, dict): + if data.get('redirect', False): + redirect = {} + if isinstance(data['redirect'], str): + redirects = [{data['state']: data['redirect']}] + elif isinstance(data['redirect'], list): + redirects = data['redirect'] + for redirect in redirects: + for mod in redirect: + funcs = salt.loader.raw_mod(self.opts, + redirect[mod], + self.functions) + if funcs: + for func in funcs: + self.functions['{0}{1}'.format(mod, func[func.rindex('.'):])] = funcs[func] self.states = salt.loader.states(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) @@ -551,12 +566,15 @@ def call(self, data): data ) ) + if 'redirect' in data: + self.load_modules(data) cdata = self.format_call(data) ret = self.states[cdata['full']](*cdata['args']) ret['__run_num__'] = self.__run_num self.__run_num += 1 format_log(ret) - self.module_refresh(data) + if 'redirect' in data: + self.load_modules() return ret def call_chunks(self, chunks): @@ -856,6 +874,8 @@ def get_tops(self): if env in include: include.pop(env) + import pprint + pprint.pprint(tops) return tops def merge_tops(self, tops): From aeaca38e3c63a6ea7d26d927295d34f4a469cde9 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 21 Feb 2012 20:33:08 -0700 Subject: [PATCH 060/598] cleaned up a some code from the previous commit --- salt/state.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/state.py b/salt/state.py index 504e8d54cb36..955dee8c3367 100644 --- a/salt/state.py +++ b/salt/state.py @@ -143,7 +143,11 @@ def load_modules(self, data=None): self.functions) if funcs: for func in funcs: - self.functions['{0}{1}'.format(mod, func[func.rindex('.'):])] = funcs[func] + f_key = '{0}{1}'.format( + mod, + func[func.rindex('.'):] + ) + self.functions[f_key] = funcs[func] self.states = salt.loader.states(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) @@ -874,8 +878,6 @@ def get_tops(self): if env in include: include.pop(env) - import pprint - pprint.pprint(tops) return tops def merge_tops(self, tops): From 11dd0ce459ea8ee1f7b0fdce726953b00b18f5c8 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Tue, 21 Feb 2012 22:02:56 -0700 Subject: [PATCH 061/598] Initial win_file.py --- salt/modules/file.py | 14 + salt/modules/win_file.py | 557 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 571 insertions(+) create mode 100644 salt/modules/win_file.py diff --git a/salt/modules/file.py b/salt/modules/file.py index d28a73c7cb7c..506d66837fd8 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -15,6 +15,20 @@ import salt.utils.find from salt.exceptions import SaltInvocationError +def __virtual__(): + ''' + Only work on posix-like systems + ''' + + # Disable on these platforms, specific file modules exist: + disable = [ + 'Windows', + ] + if __grains__['os'] in disable: + return False + return 'file' + + __outputter__ = { 'touch': 'txt', 'append': 'txt', diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py new file mode 100644 index 000000000000..648487ab2ef7 --- /dev/null +++ b/salt/modules/win_file.py @@ -0,0 +1,557 @@ +''' +Manage information about files on the minion, set/read user, group, and mode +data +''' + +# TODO: We should add the capability to do u+r type operations here +# some time in the future + +import os +import time +import hashlib +import win32api +import win32con +import win32security + +#import salt.utils.find +from salt.exceptions import SaltInvocationError + +def __virtual__(): + ''' + Only works on Windows systems + ''' + if __grains__['os'] == 'Windows': + return 'file' + return False + +__outputter__ = { + 'touch': 'txt', + 'append': 'txt', +} + + +def gid_to_group(gid): + ''' + Convert the group id to the group name on this system + + CLI Example:: + + salt '*' file.gid_to_group 0 + ''' + sid = win32security.GetBinarySid(gid) + name, domain, type = win32security.LookupAccountSid (None, sid) + return name + + +def group_to_gid(group): + ''' + Convert the group to the gid on this system + + CLI Example:: + + salt '*' file.group_to_gid root + ''' + sid, domain, type = win32security.LookupAccountName (None, group) + return win32security.ConvertSidToStringSid(sid) + + +def get_gid(path): + ''' + Return the id of the group that owns a given file + + CLI Example:: + + salt '*' file.get_gid /etc/passwd + ''' + if not os.path.exists(path): + return -1 + secdesc = win32security.GetFileSecurity (path, win32security.OWNER_SECURITY_INFORMATION) + owner_sid = secdesc.GetSecurityDescriptorOwner() + return win32security.ConvertSidToStringSid(owner_sid) + + +def get_group(path): + ''' + Return the group that owns a given file + + CLI Example:: + + salt '*' file.get_group /etc/passwd + ''' + #groupname = win32api.GetGroupNameEx(win32con.NameSamCompatible).split('\\') + #if groupname[1]: + #return groupname[1] + #return False + return 'agroup' + + +def uid_to_user(uid): + ''' + Convert a uid to a user name + + CLI Example:: + + salt '*' file.uid_to_user 0 + ''' + try: + return pwd.getpwuid(uid).pw_name + except KeyError: + return '' + + +def user_to_uid(user): + ''' + Convert user name to a uid + + CLI Example:: + + salt '*' file.user_to_uid root + ''' + try: + return pwd.getpwnam(user).pw_uid + except KeyError: + return '' + + +def get_uid(path): + ''' + Return the id of the user that owns a given file + + CLI Example:: + + salt '*' file.get_uid /etc/passwd + ''' + if not os.path.exists(path): + return False + return os.stat(path).st_uid + + +def get_user(path): + ''' + Return the user that owns a given file + + CLI Example:: + + salt '*' file.get_user /etc/passwd + ''' + username = win32api.GetUserNameEx(win32con.NameSamCompatible).split('\\') + if username[1]: + return username[1] + return False + + +def get_mode(path): + ''' + Return the mode of a file + + CLI Example:: + + salt '*' file.get_mode /etc/passwd + ''' + if not os.path.exists(path): + return -1 + mode = str(oct(os.stat(path).st_mode)[-4:]) + if mode.startswith('0'): + return mode[1:] + return mode + + +def set_mode(path, mode): + ''' + Set the mode of a file + + CLI Example:: + + salt '*' file.set_mode /etc/passwd 0644 + ''' + mode = str(mode) + if not os.path.exists(path): + return 'File not found' + try: + os.chmod(path, int(mode, 8)) + # FIXME: don't use a catch-all, be more specific... + except: + return 'Invalid Mode ' + mode + return get_mode(path) + + +def chown(path, user, group): + ''' + Chown a file, pass the file the desired user and group + + CLI Example:: + + salt '*' file.chown /etc/passwd root root + ''' + uid = user_to_uid(user) + gid = group_to_gid(group) + err = '' + if uid == '': + err += 'User does not exist\n' + if gid == '': + err += 'Group does not exist\n' + if not os.path.exists(path): + err += 'File not found' + if err: + return err + return os.chown(path, uid, gid) + + +def chgrp(path, group): + ''' + Change the group of a file + + CLI Example:: + + salt '*' file.chgrp /etc/passwd root + ''' + gid = group_to_gid(group) + err = '' + if gid == '': + err += 'Group does not exist\n' + if not os.path.exists(path): + err += 'File not found' + if err: + return err + user = get_user(path) + return chown(path, user, group) + + +def get_sum(path, form='md5'): + ''' + Return the sum for the given file, default is md5, sha1, sha224, sha256, + sha384, sha512 are supported + + CLI Example:: + + salt '*' file.get_sum /etc/passwd sha512 + ''' + if not os.path.isfile(path): + return 'File not found' + try: + return getattr(hashlib, form)(open(path, 'rb').read()).hexdigest() + except (IOError, OSError), e: + return 'File Error: %s' % (str(e)) + except AttributeError, e: + return 'Hash ' + form + ' not supported' + except NameError, e: + return 'Hashlib unavailable - please fix your python install' + except Exception, e: + return str(e) + + +def find(path, *opts): + ''' + Approximate the Unix find(1) command and return a list of paths that + meet the specified critera. + + The options include match criteria:: + + name = path-glob # case sensitive + iname = path-glob # case insensitive + regex = path-regex # case sensitive + iregex = path-regex # case insensitive + type = file-types # match any listed type + user = users # match any listed user + group = groups # match any listed group + size = [+-]number[size-unit] # default unit = byte + mtime = interval # modified since date + grep = regex # search file contents + + and/or actions:: + + delete [= file-types] # default type = 'f' + exec = command [arg ...] # where {} is replaced by pathname + print [= print-opts] + + The default action is 'print=path'. + + file-glob:: + + * = match zero or more chars + ? = match any char + [abc] = match a, b, or c + [!abc] or [^abc] = match anything except a, b, and c + [x-y] = match chars x through y + [!x-y] or [^x-y] = match anything except chars x through y + {a,b,c} = match a or b or c + + path-regex: a Python re (regular expression) pattern to match pathnames + + file-types: a string of one or more of the following:: + + a: all file types + b: block device + c: character device + d: directory + p: FIFO (named pipe) + f: plain file + l: symlink + s: socket + + users: a space and/or comma separated list of user names and/or uids + + groups: a space and/or comma separated list of group names and/or gids + + size-unit:: + + b: bytes + k: kilobytes + m: megabytes + g: gigabytes + t: terabytes + + interval:: + + [w] [[d]] [h] [m] [s] + + where: + w: week + d: day + h: hour + m: minute + s: second + + print-opts: a comma and/or space separated list of one or more of the + following:: + + group: group name + md5: MD5 digest of file contents + mode: file permissions (as integer) + mtime: last modification time (as time_t) + name: file basename + path: file absolute path + size: file size in bytes + type: file type + user: user name + + CLI Examples:: + + salt '*' file.find / type=f name=\*.bak size=+10m + salt '*' file.find /var mtime=+30d size=+10m print=path,size,mtime + salt '*' file.find /var/log name=\*.[0-9] mtime=+30d size=+10m delete + ''' + opts_dict = {} + for opt in opts: + key, value = opt.split('=', 1) + opts_dict[key] = value + try: + f = salt.utils.find.Finder(opts_dict) + except ValueError, ex: + return 'error: {0}'.format(ex) + + ret = [p for p in f.find(path)] + ret.sort() + return ret + +def _sed_esc(s): + ''' + Escape single quotes and forward slashes + ''' + return '{0}'.format(s).replace("'", "'\"'\"'").replace("/", "\/") + +def sed(path, before, after, limit='', backup='.bak', options='-r -e', + flags='g'): + ''' + Make a simple edit to a file + + Equivalent to:: + + sed "// s/// " + + path + The full path to the file to be edited + before + A pattern to find in order to replace with ``after`` + after + Text that will replace ``before`` + limit : ``''`` + An initial pattern to search for before searching for ``before`` + backup : ``.bak`` + The file will be backed up before edit with this file extension; + **WARNING:** each time ``sed``/``comment``/``uncomment`` is called will + overwrite this backup + options : ``-r -e`` + Options to pass to sed + flags : ``g`` + Flags to modify the sed search; e.g., ``i`` for case-insensitve pattern + matching + + Forward slashes and single quotes will be escaped automatically in the + ``before`` and ``after`` patterns. + + Usage:: + + salt '*' file.sed /etc/httpd/httpd.conf 'LogLevel warn' 'LogLevel info' + + .. versionadded:: 0.9.5 + ''' + # Largely inspired by Fabric's contrib.files.sed() + + before = _sed_esc(before) + after = _sed_esc(after) + + cmd = r"sed {backup}{options} '{limit}s/{before}/{after}/{flags}' {path}".format( + backup = '-i{0} '.format(backup) if backup else '', + options = options, + limit = '/{0}/ '.format(limit) if limit else '', + before = before, + after = after, + flags = flags, + path = path) + + return __salt__['cmd.run'](cmd) + +def uncomment(path, regex, char='#', backup='.bak'): + ''' + Uncomment specified commented lines in a file + + path + The full path to the file to be edited + regex + A regular expression used to find the lines that are to be uncommented. + This regex should not include the comment character. A leading ``^`` + character will be stripped for convenience (for easily switching + between comment() and uncomment()). + char : ``#`` + The character to remove in order to uncomment a line; if a single + whitespace character follows the comment it will also be removed + backup : ``.bak`` + The file will be backed up before edit with this file extension; + **WARNING:** each time ``sed``/``comment``/``uncomment`` is called will + overwrite this backup + + Usage:: + + salt '*' file.uncomment /etc/hosts.deny 'ALL: PARANOID' + + .. versionadded:: 0.9.5 + ''' + # Largely inspired by Fabric's contrib.files.uncomment() + + return __salt__['file.sed'](path, + before=r'^([[:space:]]*){0}[[:space:]]?'.format(char), + after=r'\1', + limit=regex.lstrip('^'), + backup=backup) + +def comment(path, regex, char='#', backup='.bak'): + ''' + Comment out specified lines in a file + + path + The full path to the file to be edited + regex + A regular expression used to find the lines that are to be commented; + this pattern will be wrapped in parenthesis and will move any + preceding/trailing ``^`` or ``$`` characters outside the parenthesis + (e.g., the pattern ``^foo$`` will be rewritten as ``^(foo)$``) + char : ``#`` + The character to be inserted at the beginning of a line in order to + comment it out + backup : ``.bak`` + The file will be backed up before edit with this file extension + + .. warning:: + + This backup will be overwritten each time ``sed`` / ``comment`` / + ``uncomment`` is called. Meaning the backup will only be useful + after the first invocation. + + Usage:: + + salt '*' file.comment /etc/modules pcspkr + + .. versionadded:: 0.9.5 + ''' + # Largely inspired by Fabric's contrib.files.comment() + + regex = "{0}({1}){2}".format( + '^' if regex.startswith('^') else '', + regex.lstrip('^').rstrip('$'), + '$' if regex.endswith('$') else '') + + return __salt__['file.sed']( + path, + before=regex, + after=r'{0}\1'.format(char), + backup=backup) + +def contains(path, text, limit=''): + ''' + Return True if the file at ``path`` contains ``text`` + + Usage:: + + salt '*' file.contains /etc/crontab 'mymaintenance.sh' + + .. versionadded:: 0.9.5 + ''' + # Largely inspired by Fabric's contrib.files.contains() + + if not os.path.exists(path): + return False + + result = __salt__['file.sed'](path, text, '&', limit=limit, backup='', + options='-n -r -e', flags='gp') + + return bool(result) + +def append(path, *args): + ''' + Append text to the end of a file + + Usage:: + + salt '*' file.append /etc/motd \\ + "With all thine offerings thou shalt offer salt."\\ + "Salt is what makes things taste bad when it isn't in them." + + .. versionadded:: 0.9.5 + ''' + # Largely inspired by Fabric's contrib.files.append() + + with open(path, "a") as f: + for line in args: + f.write('{0}\n'.format(line)) + + return "Wrote {0} lines to '{1}'".format(len(args), path) + +def touch(name, atime=None, mtime=None): + ''' + Just like 'nix's "touch" command, create a file if it + doesn't exist or simply update the atime and mtime if + it already does. + + atime: + Access time in Unix epoch time + mtime: + Last modification in Unix epoch time + + Usage:: + salt '*' file.touch /var/log/emptyfile + + .. versionadded:: 0.9.5 + ''' + if atime and atime.isdigit(): + atime = int(atime) + if mtime and mtime.isdigit(): + mtime = int(mtime) + try: + with open(name, "a"): + if not atime and not mtime: + times = None + elif not mtime and atime: + times = (atime, time.time()) + elif not atime and mtime: + times = (time.time(), mtime) + else: + times = (atime, mtime) + os.utime(name, times) + except TypeError as exc: + msg = "atime and mtime must be integers" + raise SaltInvocationError(msg) + except (IOError, OSError) as exc: + return False + + return os.path.exists(name) From c0a1d41295b78781a6cf5a5de011dabfa9d338e8 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Wed, 22 Feb 2012 00:57:59 -0700 Subject: [PATCH 062/598] Updated most of the file functions for Windows. --- salt/modules/win_file.py | 110 +++++++++++++++------------------------ 1 file changed, 41 insertions(+), 69 deletions(-) diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py index 648487ab2ef7..dc55019b0fa5 100644 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -1,17 +1,16 @@ ''' -Manage information about files on the minion, set/read user, group, and mode +Manage information about files on the minion, set/read user, group data ''' -# TODO: We should add the capability to do u+r type operations here -# some time in the future - import os +import os.path import time import hashlib import win32api import win32con import win32security +import ntsecuritycon as con #import salt.utils.find from salt.exceptions import SaltInvocationError @@ -36,7 +35,7 @@ def gid_to_group(gid): CLI Example:: - salt '*' file.gid_to_group 0 + salt '*' file.gid_to_group S-1-5-21-626487655-2533044672-482107328-1010 ''' sid = win32security.GetBinarySid(gid) name, domain, type = win32security.LookupAccountSid (None, sid) @@ -49,7 +48,7 @@ def group_to_gid(group): CLI Example:: - salt '*' file.group_to_gid root + salt '*' file.group_to_gid administrators ''' sid, domain, type = win32security.LookupAccountName (None, group) return win32security.ConvertSidToStringSid(sid) @@ -61,10 +60,10 @@ def get_gid(path): CLI Example:: - salt '*' file.get_gid /etc/passwd + salt '*' file.get_gid c:\\temp\\test.txt ''' if not os.path.exists(path): - return -1 + return False secdesc = win32security.GetFileSecurity (path, win32security.OWNER_SECURITY_INFORMATION) owner_sid = secdesc.GetSecurityDescriptorOwner() return win32security.ConvertSidToStringSid(owner_sid) @@ -76,13 +75,14 @@ def get_group(path): CLI Example:: - salt '*' file.get_group /etc/passwd + salt '*' file.get_group c:\\temp\\test.txt ''' - #groupname = win32api.GetGroupNameEx(win32con.NameSamCompatible).split('\\') - #if groupname[1]: - #return groupname[1] - #return False - return 'agroup' + if not os.path.exists(path): + return False + secdesc = win32security.GetFileSecurity (path, win32security.OWNER_SECURITY_INFORMATION) + owner_sid = secdesc.GetSecurityDescriptorOwner() + name, domain, type = win32security.LookupAccountSid (None, owner_sid) + return name def uid_to_user(uid): @@ -91,12 +91,11 @@ def uid_to_user(uid): CLI Example:: - salt '*' file.uid_to_user 0 + salt '*' file.uid_to_user S-1-5-21-626487655-2533044672-482107328-1010 ''' - try: - return pwd.getpwuid(uid).pw_name - except KeyError: - return '' + sid = win32security.GetBinarySid(uid) + name, domain, type = win32security.LookupAccountSid (None, sid) + return name def user_to_uid(user): @@ -105,12 +104,10 @@ def user_to_uid(user): CLI Example:: - salt '*' file.user_to_uid root + salt '*' file.user_to_uid myusername ''' - try: - return pwd.getpwnam(user).pw_uid - except KeyError: - return '' + sid, domain, type = win32security.LookupAccountName (None, user) + return win32security.ConvertSidToStringSid(sid) def get_uid(path): @@ -119,11 +116,13 @@ def get_uid(path): CLI Example:: - salt '*' file.get_uid /etc/passwd + salt '*' file.get_uid c:\\temp\\test.txt ''' if not os.path.exists(path): return False - return os.stat(path).st_uid + secdesc = win32security.GetFileSecurity (path, win32security.OWNER_SECURITY_INFORMATION) + owner_sid = secdesc.GetSecurityDescriptorOwner() + return win32security.ConvertSidToStringSid(owner_sid) def get_user(path): @@ -132,47 +131,12 @@ def get_user(path): CLI Example:: - salt '*' file.get_user /etc/passwd + salt '*' file.get_user c:\\temp\\test.txt ''' - username = win32api.GetUserNameEx(win32con.NameSamCompatible).split('\\') - if username[1]: - return username[1] - return False - - -def get_mode(path): - ''' - Return the mode of a file - - CLI Example:: - - salt '*' file.get_mode /etc/passwd - ''' - if not os.path.exists(path): - return -1 - mode = str(oct(os.stat(path).st_mode)[-4:]) - if mode.startswith('0'): - return mode[1:] - return mode - - -def set_mode(path, mode): - ''' - Set the mode of a file - - CLI Example:: - - salt '*' file.set_mode /etc/passwd 0644 - ''' - mode = str(mode) - if not os.path.exists(path): - return 'File not found' - try: - os.chmod(path, int(mode, 8)) - # FIXME: don't use a catch-all, be more specific... - except: - return 'Invalid Mode ' + mode - return get_mode(path) + secdesc = win32security.GetFileSecurity (path, win32security.OWNER_SECURITY_INFORMATION) + owner_sid = secdesc.GetSecurityDescriptorOwner() + name, domain, type = win32security.LookupAccountSid (None, owner_sid) + return name def chown(path, user, group): @@ -181,8 +145,10 @@ def chown(path, user, group): CLI Example:: - salt '*' file.chown /etc/passwd root root + salt '*' file.chown c:\\temp\\test.txt myusername administrators ''' + # I think this function isn't working correctly yet + sd = win32security.GetFileSecurity (path, win32security.DACL_SECURITY_INFORMATION) uid = user_to_uid(user) gid = group_to_gid(group) err = '' @@ -194,7 +160,12 @@ def chown(path, user, group): err += 'File not found' if err: return err - return os.chown(path, uid, gid) + + dacl = win32security.ACL () + dacl.AddAccessAllowedAce (win32security.ACL_REVISION, con.FILE_ALL_ACCESS, win32security.GetBinarySid(uid)) + dacl.AddAccessAllowedAce (win32security.ACL_REVISION, con.FILE_ALL_ACCESS, win32security.GetBinarySid(gid)) + sd.SetSecurityDescriptorDacl (1, dacl, 0) + return win32security.SetFileSecurity (path, win32security.DACL_SECURITY_INFORMATION, sd) def chgrp(path, group): @@ -203,8 +174,9 @@ def chgrp(path, group): CLI Example:: - salt '*' file.chgrp /etc/passwd root + salt '*' file.chgrp c:\\temp\\test.txt administrators ''' + # I think this function isn't working correctly yet gid = group_to_gid(group) err = '' if gid == '': From 9b966a57fe62ee53eced538f465d8cf4c9d994a9 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 22 Feb 2012 01:25:39 -0700 Subject: [PATCH 063/598] Add iter methods to the client module The iter methods allow Salt to return data from the minions as it arrives, this makes Salt better at quickly returning data to the user and makes for an interface which can be used to fater the data in a more fluid manner. The iter functions are generators --- salt/client.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/salt/client.py b/salt/client.py index bee273603f55..548b729f1f1c 100644 --- a/salt/client.py +++ b/salt/client.py @@ -151,6 +151,33 @@ def cmd( timeout=timeout) return self.get_returns(pub_data['jid'], pub_data['minions'], timeout) + def cmd_iter( + self, + tgt, + fun, + arg=(), + timeout=None, + expr_form='glob', + ret=''): + ''' + Execute a salt command and return + ''' + if timeout is None: + timeout = self.opts['timeout'] + jid = prep_jid(self.opts['cachedir']) + pub_data = self.pub( + tgt, + fun, + arg, + expr_form, + ret, + jid=jid, + timeout=timeout) + for fn_ret in self.get_iter_returns(pub_data['jid'], + pub_data['minions'], + timeout): + yield fn_ret + def cmd_full_return( self, tgt, @@ -176,6 +203,53 @@ def cmd_full_return( return (self.get_full_returns(pub_data['jid'], pub_data['minions'], timeout)) + def get_iter_returns(self, jid, minions, timeout=None): + ''' + This method starts off a watcher looking at the return data for a + specified jid, it returns all of the information for the jid + ''' + if timeout is None: + timeout = self.opts['timeout'] + jid_dir = os.path.join(self.opts['cachedir'], 'jobs', jid) + start = 999999999999 + gstart = int(time.time()) + found = set() + # Check to see if the jid is real, if not return the empty dict + if not os.path.isdir(jid_dir): + yield {} + # Wait for the hosts to check in + while True: + ret = {} + for fn_ in os.listdir(jid_dir): + if fn_.startswith('.'): + continue + if fn_ not in found: + retp = os.path.join(jid_dir, fn_, 'return.p') + outp = os.path.join(jid_dir, fn_, 'out.p') + if not os.path.isfile(retp): + continue + while fn_ not in ret: + try: + ret_data = self.serial.load(open(retp, 'r')) + ret[fn_] = {'ret': ret_data} + if os.path.isfile(outp): + ret[fn_]['out'] = self.serial.load(open(outp, 'r')) + except: + pass + found.add(fn_) + yield ret + if ret and start == 999999999999: + start = int(time.time()) + if len(ret) >= len(minions): + break + if int(time.time()) > start + timeout: + break + if int(time.time()) > gstart + timeout and not ret: + # No minions have replied within the specified global timeout, + # return an empty dict + break + time.sleep(0.02) + def get_returns(self, jid, minions, timeout=None): ''' This method starts off a watcher looking at the return data for a From 5f7d69100d3bc6ccda336c838122d845bcd67881 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 22 Feb 2012 01:27:44 -0700 Subject: [PATCH 064/598] Added the -i option to run Salt in iter mode iter mode uses the new iter methods to return data to the terminal as it arives, this makes the command execution much more fluid --- salt/cli/__init__.py | 72 ++++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 116666abd817..a448e69e4d3f 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -43,6 +43,13 @@ def __parse(self): dest='timeout', help=('Set the return timeout for batch jobs; ' 'default=5 seconds')) + parser.add_option('-i', + '--iter', + '--iter-return', + default=False, + dest='iter_', + action='store_true', + help='Return the data from minions as the data is returned') parser.add_option('-E', '--pcre', default=False, @@ -145,6 +152,7 @@ def __parse(self): if not options.timeout is None: opts['timeout'] = int(options.timeout) + opts['iter'] = options.iter_ opts['pcre'] = options.pcre opts['list'] = options.list_ opts['grain'] = options.grain @@ -252,39 +260,45 @@ def run(self): try: # local will be None when there was an error if local: - full_ret = local.cmd_full_return(*args) - ret, out = self._format_ret(full_ret) + if self.opts['iter']: + for full_ret in local.cmd_iter(*args): + ret, out = self._format_ret(full_ret) + self._output_ret(ret, out) + else: + full_ret = local.cmd_full_return(*args) + ret, out = self._format_ret(full_ret) + self._output_ret(ret, out) except SaltInvocationError as exc: ret = exc out = '' - # Handle special case commands - if self.opts['fun'] == 'sys.doc': - self._print_docs(ret) - else: - # Determine the proper output method and run it - get_outputter = salt.output.get_outputter - if isinstance(ret, list) or isinstance(ret, dict): - if self.opts['raw_out']: - printout = get_outputter('raw') - elif self.opts['json_out']: - printout = get_outputter('json') - elif self.opts['txt_out']: - printout = get_outputter('txt') - elif self.opts['yaml_out']: - printout = get_outputter('yaml') - elif out: - printout = get_outputter(out) - else: - printout = get_outputter(None) - # Pretty print any salt exceptions - elif isinstance(ret, SaltException): - printout = get_outputter("txt") - printout(ret) - - # Always exit with a return code of 1 on issues - if isinstance(ret, Exception): - sys.exit(1) + def _output_ret(self, ret, out): + ''' + Print the output from a single return to the terminal + ''' + # Handle special case commands + if self.opts['fun'] == 'sys.doc': + self._print_docs(ret) + else: + # Determine the proper output method and run it + get_outputter = salt.output.get_outputter + if isinstance(ret, list) or isinstance(ret, dict): + if self.opts['raw_out']: + printout = get_outputter('raw') + elif self.opts['json_out']: + printout = get_outputter('json') + elif self.opts['txt_out']: + printout = get_outputter('txt') + elif self.opts['yaml_out']: + printout = get_outputter('yaml') + elif out: + printout = get_outputter(out) + else: + printout = get_outputter(None) + # Pretty print any salt exceptions + elif isinstance(ret, SaltException): + printout = get_outputter("txt") + printout(ret) def _format_ret(self, full_ret): ''' From f15eb64276d58d8288d9321fb1a005a43c8ffde5 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 22 Feb 2012 01:37:29 -0700 Subject: [PATCH 065/598] initialize the ret value in the correct scope --- salt/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/client.py b/salt/client.py index 548b729f1f1c..9c41a39c713c 100644 --- a/salt/client.py +++ b/salt/client.py @@ -219,8 +219,8 @@ def get_iter_returns(self, jid, minions, timeout=None): yield {} # Wait for the hosts to check in while True: - ret = {} for fn_ in os.listdir(jid_dir): + ret = {} if fn_.startswith('.'): continue if fn_ not in found: From e02197baefce111ed6fd8574aa47136b42c2dcba Mon Sep 17 00:00:00 2001 From: Ben Hosmer Date: Wed, 22 Feb 2012 07:59:23 -0500 Subject: [PATCH 066/598] adding the option to delete all keys using salt-key option --- salt/cli/__init__.py | 8 ++++++++ salt/cli/key.py | 28 ++++++++++++++++++++-------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 116666abd817..c120e92c5619 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -478,6 +478,13 @@ def __parse(self): dest='delete', default='', help='Delete the named key') + + parser.add_option('-D', + '--delete-all', + dest='delete_all', + default=False, + action='store_true', + help='Delete all keys') parser.add_option('-q', '--quiet', @@ -536,6 +543,7 @@ def __parse(self): opts['print'] = options.print_ opts['print_all'] = options.print_all opts['delete'] = options.delete + opts['delete_all'] = options.delete_all opts['gen_keys'] = options.gen_keys opts['gen_keys_dir'] = options.gen_keys_dir if options.keysize < 2048: diff --git a/salt/cli/key.py b/salt/cli/key.py index 98a97e76c0d5..230df0de3425 100644 --- a/salt/cli/key.py +++ b/salt/cli/key.py @@ -143,28 +143,38 @@ def _accept_all(self): for key in os.listdir(minions_pre): self._accept(key) - def _delete_key(self): + def _delete_key(self, delete=None): ''' Delete a key ''' (minions_accepted, minions_pre, minions_rejected) = self._check_minions_directories() - pre = os.path.join(minions_pre, self.opts['delete']) - acc = os.path.join(minions_accepted, self.opts['delete']) - rej= os.path.join(minions_rejected, self.opts['delete']) + if delete == None: + delete = self.opts['delete'] + pre = os.path.join(minions_pre, delete) + acc = os.path.join(minions_accepted, delete) + rej = os.path.join(minions_rejected, delete) if os.path.exists(pre): os.remove(pre) - self._log('Removed pending key %s' % self.opts['delete'], + self._log('Removed pending key %s' % delete, level='info') if os.path.exists(acc): os.remove(acc) - self._log('Removed accepted key %s' % self.opts['delete'], + self._log('Removed accepted key %s' % delete, level='info') if os.path.exists(rej): os.remove(rej) - self._log('Removed rejected key %s' % self.opts['delete'], + self._log('Removed rejected key %s' % delete, level='info') + def _delete_all(self): + ''' + Delete all keys + ''' + for dir in ("acc", "rej", "pre"): + for key in self._keys(dir): + self._delete_key(key) + def _reject(self, key): ''' @@ -234,5 +244,7 @@ def run(self): self._reject_all() elif self.opts['delete']: self._delete_key() + elif self.opts['delete_all']: + self._delete_all() else: - self._list_all() + self._list_all() \ No newline at end of file From 6dc7ab29ada8d47bf975f6c9bd7702b44100abad Mon Sep 17 00:00:00 2001 From: Evan Borgstrom Date: Wed, 22 Feb 2012 21:43:58 -0500 Subject: [PATCH 067/598] Start of parse_network work, but now I'm stuck. Off to github to open an issue... --- salt/modules/network.py | 49 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/salt/modules/network.py b/salt/modules/network.py index d7c67ed8ff5e..46f5454bd860 100644 --- a/salt/modules/network.py +++ b/salt/modules/network.py @@ -170,25 +170,59 @@ def interfaces(): for group in groups: iface = None + parent = None up = False for line in group.split('\n'): if not ' ' in line: continue - m = re.match('^\d*:\s+(\w+):\s+<(.+)>', line) + m = re.match('^\d*:\s+(\w+)(@\w+)?:\s+<(.+)>', line) if m: - iface,attrs = m.groups() + iface,parent,attrs = m.groups() if 'UP' in attrs.split(','): up = True ipaddr = None + secondary = [] netmask = None + broadcast = None hwaddr = None else: cols = line.split() if len(cols) >= 2: type,value = tuple(cols[0:2]) - if type == 'inet': - ipaddr,cidr = tuple(value.split('/')) - netmask = _cidr_to_ipv4_netmask(int(cidr)) + if type.startswith('inet'): + def parse_network(iface): + """ + Return a tuple of ip, netmask, broadcast + based on the current set of cols + + This is so we can handle secondary addresses + with the same logic as primary + """ + ip,cidr = tuple(value.split('/')) + if type == 'inet': + mask = _cidr_to_ipv4_netmask(int(cidr)) + elif type == 'inet6': + mask = cidr + else: + raise RuntimeError("Don't know how to handle netmask for interface type %s on %s" % (type, iface)) + + try: + brd = cols[cols.index('brd')+1] + except ValueError: + brd = None + + return (ip, mask, brd) + + if 'secondary' not in cols: + ipaddr, netmask, broadcast = parse_network(iface) + else: + ip, mask, brd = parse_network(iface) + secondary.append({ + 'ipaddr': ip, + 'netmask': mask, + 'broadcast': brd + }) + del ip, mask, brd elif type.startswith('link'): hwaddr = value if iface: @@ -196,7 +230,12 @@ def interfaces(): ret[iface]['up'] = up ret[iface]['ipaddr'] = ipaddr ret[iface]['netmask'] = netmask + ret[iface]['broadcast'] = broadcast ret[iface]['hwaddr'] = hwaddr + if parent is not None: + ret[iface]['parent'] = parent + if len(secondary) > 0: + ret[iface]['secondary'] = secondary del iface,up return ret From 1cd0b22c57cc3acc371483887dd8b28efa878701 Mon Sep 17 00:00:00 2001 From: Anselm Helbig Date: Thu, 23 Feb 2012 15:09:14 +0100 Subject: [PATCH 068/598] added modules and state modules for rubygem and rvm support; see #667 --- salt/modules/gem.py | 131 ++++++++++++++++++++++++ salt/modules/rvm.py | 235 ++++++++++++++++++++++++++++++++++++++++++++ salt/states/gem.py | 70 +++++++++++++ salt/states/rvm.py | 221 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 657 insertions(+) create mode 100644 salt/modules/gem.py create mode 100644 salt/modules/rvm.py create mode 100644 salt/states/gem.py create mode 100644 salt/states/rvm.py diff --git a/salt/modules/gem.py b/salt/modules/gem.py new file mode 100644 index 000000000000..62a869cb5869 --- /dev/null +++ b/salt/modules/gem.py @@ -0,0 +1,131 @@ +""" +Manage ruby gems. +""" + +__opts__ = { + "rvm.runas": None, +} + +import re + +def _gem(command, ruby = None, runas = None): + cmdline = "gem {command}".format(command = command) + if __salt__["rvm.is_installed"]: + return __salt__["rvm.do"](ruby, cmdline, runas = runas) + + ret = __salt__["cmd.run_all"](cmdline, runas = runas or __opts__["rvm.runas"]) + + if ret["retcode"] == 0: + return ret["stdout"] + else: + return False + + +def install(gems, ruby = None, runas = None): + """ + Installs one or several gems. + + gems + The gems to install. + ruby : None + If RVM is installed, the ruby version and gemset to use. + runas : None + The user to run gem as. + """ + return _gem("install {gems}".format(gems = gems), ruby, runas = runas) + +def uninstall(gems, ruby = None, runas = None): + """ + Uninstall one or several gems. + + gems + The gems to uninstall. + ruby : None + If RVM is installed, the ruby version and gemset to use. + runas : None + The user to run gem as. + """ + return _gem("uninstall {gems}".format(gems = gems), ruby, runas = runas) + +def update(gems, ruby = None, runas = None): + """ + Update one or several gems. + + gems + The gems to update. + ruby : None + If RVM is installed, the ruby version and gemset to use. + runas : None + The user to run gem as. + """ + return _gem("update {gems}".format(gems = gems), ruby, runas = runas) + +def update_system(version = "", ruby = None, runas = None): + """ + Update rubygems. + + version : (newest) + The version of rubygems to install. + ruby : None + If RVM is installed, the ruby version and gemset to use. + runas : None + The user to run gem as. + """ + return _gem("update --system {version}".format(version = version), ruby, runas = runas) + +def list(prefix = "", ruby = None, runas = None): + """ + List locally installed gems. + + prefix : + Only list gems when the name matches this prefix. + ruby : None + If RVM is installed, the ruby version and gemset to use. + runas : None + The user to run gem as. + """ + gems = {} + for line in _gem("list {prefix}".format(prefix = prefix), ruby, runas = runas).splitlines(): + m = re.match("^([^ ]+) \((.+)\)", line) + if m: + gem = m.group(1) + versions = m.group(2).split(", ") + gems[gem] = versions + return gems + +def sources_add(source_uri, ruby = None, runas = None): + """ + Add a gem source. + + source_uri + The source URI to add. + ruby : None + If RVM is installed, the ruby version and gemset to use. + runas : None + The user to run gem as. + """ + return _gem("sources --add {source_uri}".format(source_uri = source_uri), ruby, runas = runas) + +def sources_remove(source_uri, ruby = None, runas = None): + """ + Remove a gem source. + + source_uri + The source URI to remove. + ruby : None + If RVM is installed, the ruby version and gemset to use. + runas : None + The user to run gem as. + """ + return _gem("sources --remove {source_uri}".format(source_uri = source_uri), ruby, runas = runas) + +def sources_list(ruby = None, runas = None): + """ + List the configured gem sources. + + ruby : None + If RVM is installed, the ruby version and gemset to use. + runas : None + The user to run gem as. + """ + return _gem("sources", ruby, runas = runas).splitlines()[2:] diff --git a/salt/modules/rvm.py b/salt/modules/rvm.py new file mode 100644 index 000000000000..737961fa1958 --- /dev/null +++ b/salt/modules/rvm.py @@ -0,0 +1,235 @@ +""" +Manage ruby installations and gemsets with RVM, the Ruby Version Manager. +""" + +__opts__ = { + "rvm.runas": None, +} + +import re + +def _rvm(command, arguments = "", runas = None): + if not is_installed(): + return False + + ret = __salt__["cmd.run_all"]("/usr/local/rvm/bin/rvm {command} {arguments}".format(command = command, arguments = arguments), runas = runas or __opts__["rvm.runas"]) + if ret["retcode"] == 0: + return ret["stdout"] + else: + return False + +def _rvm_do(ruby, command, runas = None): + return _rvm("{ruby} do {command}".format(ruby = ruby or "default", command = command), runas = runas) + +def is_installed(): + """ + Check if RVM is installed. + """ + return __salt__["cmd.has_exec"]("/usr/local/rvm/bin/rvm") + +# RVM dependencies on Ubuntu 10.04: +# bash coreutils gzip bzip2 gawk sed curl git-core subversion sudo +def install(): + """ + Install RVM system wide. + """ + return 0 == __salt__["cmd.retcode"]( + # the RVM installer only does a multi-user install when it is invoked under sudo + "curl -s https://raw.github.com/wayneeseguin/rvm/master/binscripts/rvm-installer | sudo bash -s stable") + +# MRI/RBX/REE dependencies for Ubuntu 10.04: +# build-essential openssl libreadline6 libreadline6-dev curl +# git-core zlib1g zlib1g-dev libssl-dev libyaml-dev libsqlite3-0 +# libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev autoconf libc6-dev +# libncurses5-dev automake libtool bison subversion ruby +def install_ruby(ruby, runas = None): + """ + Install a ruby implementation. + + ruby + The version of ruby to install. + runas : None + The user to run rvm as. + """ + return _rvm("install", ruby, runas = runas) + +def reinstall_ruby(ruby, runas = None): + """ + Reinstall a ruby implementation. + + ruby + The version of ruby to reinstall. + runas : None + The user to run rvm as. + """ + return _rvm("reinstall", ruby, runas = runas) + +def list(runas = None): + """ + List all rvm installed rubies. + + runas : None + The user to run rvm as. + """ + rubies = [] + + for line in _rvm("list", "", runas = runas).splitlines(): + m = re.match("^[= ]([*> ]) ([^- ]+)-([^ ]+) \[ (.*) \]", line) + if m: + rubies.append([m.group(2), m.group(3), m.group(1) == "*"]) + return rubies + +def set_default(ruby, runas = None): + """ + Set the default ruby. + + ruby + The version of ruby to make the default. + runas : None + The user to run rvm as. + """ + return _rvm("alias", "create default {ruby}".format(ruby = ruby), runas = runas) + +def get(version = "stable", runas = None): + """ + Update RVM. + + version : stable + Which version of RVM to install, e.g. stable or head. + ruby + The version of ruby to reinstall. + """ + return _rvm("get", version, runas = runas) + +def wrapper(ruby_string, wrapper_prefix, runas = None, *binaries): + """ + Install RVM wrapper scripts. + + ruby_string + Ruby/gemset to install wrappers for. + wrapper_prefix + What to prepend to the name of the generated wrapper binaries. + runas : None + The user to run rvm as. + binaries : None + The names of the binaries to create wrappers for. When nothing is given, wrappers for ruby, gem, rake, irb, rdoc, ri and testrb are generated. + """ + return _rvm("wrapper", "{ruby_string} {wrapper_prefix} {binaries}". + format(ruby_string = ruby_string, + wrapper_prefix = wrapper_prefix, + binaries = " ".join(binaries)), runas = runas) + +def rubygems(ruby, version, runas = None): + """ + Installs a specific rubygems version in the given ruby. + + ruby + The ruby to install rubygems for. + version + The version of rubygems to install or 'remove' to use the version that ships with 1.9 + runas : None + The user to run rvm as. + """ + return _rvm_do(ruby, "rubygems", version, runas = runas) + +def gemset_create(ruby, gemset, runas = None): + """ + Creates a gemset. + + ruby + The ruby version to create the gemset for. + gemset + The name of the gemset to create. + runas : None + The user to run rvm as. + """ + return _rvm_do(ruby, "rvm gemset create {gemset}".format(gemset = gemset), runas = runas) + +def gemset_list(ruby = "default", runas = None): + """ + List all gemsets for the given ruby. + + ruby : default + The ruby version to list the gemsets for + runas : None + The user to run rvm as. + """ + gemsets = [] + for line in _rvm_do(ruby, "rvm gemset list", runas = runas).splitlines(): + m = re.match("^ ([^ ]+)", line) + if m: + gemsets.append(m.group(1)) + return gemsets + +def gemset_delete(ruby, gemset, runas = None): + """ + Deletes a gemset. + + ruby + The ruby version the gemset belongs to. + gemset + The gemset to delete. + runas : None + The user to run rvm as. + """ + return _rvm_do(ruby, "rvm --force gemset delete {gemset}".format(gemset = gemset), runas = runas) + +def gemset_empty(ruby, gemset, runas = None): + """ + Remove all gems from a gemset. + + ruby + The ruby version the gemset belongs to. + gemset + The gemset to empty. + runas : None + The user to run rvm as. + """ + return _rvm_do(ruby, "rvm --force gemset empty", gemset, runas = runas) + +def gemset_copy(source, destination, runas = None): + """ + Copy all gems from one gemset to another. + + source + The name of the gemset to copy, complete with ruby version. + destination + The destination gemset. + runas : None + The user to run rvm as. + """ + return _rvm("gemset copy", source, destination, runas = runas) + +def gemset_list_all(runas = None): + """ + List all gemsets for all installed rubies. + + Note that you must have set a default ruby before this can work. + runas : None + The user to run rvm as. + """ + gemsets = {} + current_ruby = None + for line in _rvm_do("default", "rvm gemset list_all", runas = runas).splitlines(): + m = re.match("^gemsets for ([^ ]+)", line) + if m: + current_ruby = m.group(1) + gemsets[current_ruby] = [] + m = re.match("^ ([^ ]+)", line) + if m: + gemsets[current_ruby].append(m.group(1)) + return gemsets + +def do(ruby, command, runas = None): + """ + Execute a command in an RVM controlled environment. + + ruby: + The ruby to use. + command: + The command to execute. + runas : None + The user to run rvm as. + """ + return _rvm_do(ruby, command, runas = runas) + diff --git a/salt/states/gem.py b/salt/states/gem.py new file mode 100644 index 000000000000..422ef8a52f1d --- /dev/null +++ b/salt/states/gem.py @@ -0,0 +1,70 @@ +""" +Management of rubygems +======================= +A state module to manage rubygems. Gems can be set up to be installed +or removed. This module will use RVM if it is installed. In that case +you can specify what ruby version and gemset to target. + +.. code-block:: yaml + + addressable: + gem: + - installed + - runas: rvm + - ruby: jruby@jgemset +""" + +def installed(name, ruby = None, runas = None): + """ + Make sure that a gem is installed. + + name + The name of the gem to install + ruby : None + For RVM installations: the ruby version and gemset to target. + runas : None + The user to run gem as. + """ + ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} + if name in __salt__["gem.list"](name, ruby, runas = runas): + ret["result"] = True + ret["comment"] = "Gem is already installed." + return ret + + if __salt__["gem.install"](name, ruby, runas = runas): + ret["result"] = True + ret["changes"][name] = "Installed" + ret["comment"] = "Gem was successfully installed" + else: + ret["result"] = False + ret["comment"] = "Could not install gem." + + return ret + +def removed(name, ruby = None, runas = None): + """ + Make sure that a gem is not installed. + + name + The name of the gem to uninstall + ruby : None + For RVM installations: the ruby version and gemset to target. + runas : None + The user to run gem as. + """ + ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} + if name not in __salt__["gem.list"](name, ruby, runas = runas): + ret["result"] = True + ret["comment"] = "Gem is not installed." + return ret + + if __salt__["gem.uninstall"](name, ruby, runas = runas): + ret["result"] = True + ret["changes"][name] = "Removed" + ret["comment"] = "Gem was successfully removed." + else: + ret["result"] = False + ret["comment"] = "Could not remove gem." + return ret + + diff --git a/salt/states/rvm.py b/salt/states/rvm.py new file mode 100644 index 000000000000..c1603c51cecb --- /dev/null +++ b/salt/states/rvm.py @@ -0,0 +1,221 @@ +""" +Management of ruby installations and gemsets with RVM +===================================================== +This module is used to install and manage ruby installations and +gemsets with RVM, the Ruby Version Manager. Different versions of ruby +can be installed and gemsets created. RVM itself will be installed +automatically if it's not present. This module will not automatically +install packages that RVM depends on or ones that are needed to build +ruby. If you want to run RVM as an unprivileged user (recommended) you +will have to create this user yourself. This is how a state +configuration could look like: + +.. code-block:: yaml + +rvm: + group: + - present + user: + - present + - gid: rvm + - home: /home/rvm + - require: + - group: rvm + +rvm-deps: + pkg: + - installed + - names: + - bash + - coreutils + - gzip + - bzip2 + - gawk + - sed + - curl + - git-core + - subversion + - sudo + +mri-deps: + pkg: + - installed + - names: + - build-essential + - openssl + - libreadline6 + - libreadline6-dev + - curl + - git-core + - zlib1g + - zlib1g-dev + - libssl-dev + - libyaml-dev + - libsqlite3-0 + - libsqlite3-dev + - sqlite3 + - libxml2-dev + - libxslt1-dev + - autoconf + - libc6-dev + - libncurses5-dev + - automake + - libtool + - bison + - subversion + - ruby + +jruby-deps: + pkg: + - installed + - names: + - curl + - g++ + - openjdk-6-jre-headless + +ruby-1.9.2: + rvm: + - installed + - default: True + - runas: rvm + - require: + - pkg: rvm-deps + - pkg: mri-deps + - user: rvm + +jruby: + rvm: + - installed + - runas: rvm + - require: + - pkg: rvm-deps + - pkg: jruby-deps + - user: rvm + +jgemset: + rvm: + - gemset_present + - ruby: jruby + - runas: rvm + - require: + - rvm: jruby + +mygemset: + rvm: + - gemset_present + - ruby: ruby-1.9.2 + - runas: rvm + - require: + - rvm: ruby-1.9.2 +""" + +import re + +def _check_rvm(ret): + if not __salt__["rvm.is_installed"](): + if __salt__["rvm.install"](): + ret["changes"]["rvm"] = "Installed" + else: + ret["result"] = False + ret["comment"] = "Could not install RVM." + return ret + +def _check_and_install_ruby(ret, ruby, default = False, runas = None): + ret = _check_ruby(ret, ruby, runas = runas) + if not ret["result"]: + if __salt__["rvm.install_ruby"](ruby, runas = runas): + ret["result"] = True + ret["changes"][ruby] = "Installed" + ret["comment"] = "Successfully installed ruby." + ret["default"] = False + else: + ret["result"] = False + ret["comment"] = "Could not install ruby." + return ret + + if not ret["default"] and default: + __salt__["rvm.set_default"](ruby, runas = runas) + + return ret + +def _check_ruby(ret, ruby, runas = None): + match_version = True + match_micro_version = False + micro_version_regex = re.compile("-([0-9]{4}\.[0-9]{2}|p[0-9]+)$") + if micro_version_regex.match(ruby): + match_micro_version = True + if re.match("^[a-z]+$", ruby): + match_version = False + ruby = re.sub("^ruby-", "", ruby) + + for impl, version, default in __salt__["rvm.list"](runas = runas): + if impl != "ruby": + version = "{impl}-{version}".format(impl = impl, version = version) + if not match_micro_version: + version = micro_version_regex.sub("", version) + if not match_version: + version = re.sub("-.*", "") + if version == ruby: + ret["result"] = True + ret["comment"] = "Requested ruby exists." + ret["default"] = default + break + return ret + +def installed(name, default = False, runas = None): + """ + Verify that the specified ruby is installed with RVM. RVM is installed when necessary. + + name + The version of ruby to install + default : False + Whether to make this ruby the default. + runas : None + The user to run rvm as. + """ + ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} + + ret = _check_rvm(ret) + if ret["result"] == False: + return ret + + return _check_and_install_ruby(ret, name, default, runas = runas) + +def gemset_present(name, ruby = "default", runas = None): + """ + Verify that the gemset is present. + + name + The name of the gemset. + ruby : default + The ruby version this gemset belongs to. + runas : None + The use user to run rvm as. + """ + ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} + + ret = _check_rvm(ret) + if ret["result"] == False: + return ret + + if name.find("@") != -1: + ruby, name = name.split("@") + ret = _check_ruby(ret, ruby) + if not ret["result"]: + ret["result"] = False + ret["comment"] = "Requested ruby implementation was not found." + return ret + + if name in __salt__["rvm.gemset_list"](ruby, runas = runas): + ret["result"] = True + ret["comment"] = "Gemset already exists." + else: + if __salt__["rvm.gemset_create"](ruby, name, runas = runas): + ret["result"] = True + ret["comment"] = "Gemset successfully created." + ret["changes"][name] = "created" + else: + ret["result"] = False + ret["comment"] = "Gemset could not be created." + + return ret From 71c9845f42700ecf2cbc3f22efe7fca39fe1de14 Mon Sep 17 00:00:00 2001 From: Evan Borgstrom Date: Thu, 23 Feb 2012 09:30:50 -0500 Subject: [PATCH 069/598] Refine & refactor --- salt/modules/network.py | 63 +++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/salt/modules/network.py b/salt/modules/network.py index 46f5454bd860..15f84c067c0b 100644 --- a/salt/modules/network.py +++ b/salt/modules/network.py @@ -170,73 +170,62 @@ def interfaces(): for group in groups: iface = None - parent = None - up = False + data = {} for line in group.split('\n'): if not ' ' in line: continue - m = re.match('^\d*:\s+(\w+)(@\w+)?:\s+<(.+)>', line) + m = re.match('^\d*:\s+(\w+)(?:@)?(\w+)?:\s+<(.+)>', line) if m: iface,parent,attrs = m.groups() if 'UP' in attrs.split(','): - up = True - ipaddr = None - secondary = [] - netmask = None - broadcast = None - hwaddr = None + data['up'] = True + if parent: + data['parent'] = parent else: cols = line.split() if len(cols) >= 2: type,value = tuple(cols[0:2]) - if type.startswith('inet'): - def parse_network(iface): + if type in ('inet', 'inet6'): + def parse_network(): """ Return a tuple of ip, netmask, broadcast based on the current set of cols - - This is so we can handle secondary addresses - with the same logic as primary """ + brd = None ip,cidr = tuple(value.split('/')) if type == 'inet': mask = _cidr_to_ipv4_netmask(int(cidr)) + if 'brd' in cols: + brd = cols[cols.index('brd')+1] elif type == 'inet6': mask = cidr - else: - raise RuntimeError("Don't know how to handle netmask for interface type %s on %s" % (type, iface)) - - try: - brd = cols[cols.index('brd')+1] - except ValueError: - brd = None - return (ip, mask, brd) if 'secondary' not in cols: - ipaddr, netmask, broadcast = parse_network(iface) + ipaddr, netmask, broadcast = parse_network() + if type == 'inet': + data['ipaddr'] = ipaddr + data['netmask'] = netmask + data['broadcast'] = broadcast + elif type == 'inet6': + data['ipaddr6'] = ipaddr + data['netmask6'] = netmask else: - ip, mask, brd = parse_network(iface) - secondary.append({ + if 'secondary' not in data: + data['secondary'] = [] + ip, mask, brd = parse_network() + data['secondary'].append({ + 'type': type, 'ipaddr': ip, 'netmask': mask, 'broadcast': brd }) del ip, mask, brd elif type.startswith('link'): - hwaddr = value + data['hwaddr'] = value if iface: - ret[iface] = {} - ret[iface]['up'] = up - ret[iface]['ipaddr'] = ipaddr - ret[iface]['netmask'] = netmask - ret[iface]['broadcast'] = broadcast - ret[iface]['hwaddr'] = hwaddr - if parent is not None: - ret[iface]['parent'] = parent - if len(secondary) > 0: - ret[iface]['secondary'] = secondary - del iface,up + ret[iface] = data + del iface, data return ret From a7a40ca53947560599aede6b0baa6494997a9b13 Mon Sep 17 00:00:00 2001 From: Anselm Helbig Date: Thu, 23 Feb 2012 17:04:38 +0100 Subject: [PATCH 070/598] make gem and rvm modules conform to pep8 --- salt/modules/gem.py | 57 +++++++++++++-------- salt/modules/rvm.py | 121 ++++++++++++++++++++++++++------------------ salt/states/gem.py | 18 +++---- salt/states/rvm.py | 44 +++++++++------- 4 files changed, 141 insertions(+), 99 deletions(-) diff --git a/salt/modules/gem.py b/salt/modules/gem.py index 62a869cb5869..7a626fab72be 100644 --- a/salt/modules/gem.py +++ b/salt/modules/gem.py @@ -8,20 +8,22 @@ import re -def _gem(command, ruby = None, runas = None): - cmdline = "gem {command}".format(command = command) + +def _gem(command, ruby=None, runas=None): + cmdline = "gem {command}".format(command=command) if __salt__["rvm.is_installed"]: - return __salt__["rvm.do"](ruby, cmdline, runas = runas) + return __salt__["rvm.do"](ruby, cmdline, runas=runas) - ret = __salt__["cmd.run_all"](cmdline, runas = runas or __opts__["rvm.runas"]) + ret = __salt__["cmd.run_all"]( + cmdline, runas=runas or __opts__["rvm.runas"]) if ret["retcode"] == 0: return ret["stdout"] else: return False - -def install(gems, ruby = None, runas = None): + +def install(gems, ruby=None, runas=None): """ Installs one or several gems. @@ -32,9 +34,10 @@ def install(gems, ruby = None, runas = None): runas : None The user to run gem as. """ - return _gem("install {gems}".format(gems = gems), ruby, runas = runas) + return _gem("install {gems}".format(gems=gems), ruby, runas=runas) + -def uninstall(gems, ruby = None, runas = None): +def uninstall(gems, ruby=None, runas=None): """ Uninstall one or several gems. @@ -45,9 +48,10 @@ def uninstall(gems, ruby = None, runas = None): runas : None The user to run gem as. """ - return _gem("uninstall {gems}".format(gems = gems), ruby, runas = runas) + return _gem("uninstall {gems}".format(gems=gems), ruby, runas=runas) -def update(gems, ruby = None, runas = None): + +def update(gems, ruby=None, runas=None): """ Update one or several gems. @@ -58,9 +62,10 @@ def update(gems, ruby = None, runas = None): runas : None The user to run gem as. """ - return _gem("update {gems}".format(gems = gems), ruby, runas = runas) + return _gem("update {gems}".format(gems=gems), ruby, runas=runas) + -def update_system(version = "", ruby = None, runas = None): +def update_system(version="", ruby=None, runas=None): """ Update rubygems. @@ -71,13 +76,15 @@ def update_system(version = "", ruby = None, runas = None): runas : None The user to run gem as. """ - return _gem("update --system {version}".format(version = version), ruby, runas = runas) + return _gem("update --system {version}". + format(version=version), ruby, runas=runas) -def list(prefix = "", ruby = None, runas = None): + +def list(prefix="", ruby=None, runas=None): """ List locally installed gems. - prefix : + prefix : Only list gems when the name matches this prefix. ruby : None If RVM is installed, the ruby version and gemset to use. @@ -85,7 +92,8 @@ def list(prefix = "", ruby = None, runas = None): The user to run gem as. """ gems = {} - for line in _gem("list {prefix}".format(prefix = prefix), ruby, runas = runas).splitlines(): + for line in _gem("list {prefix}".format(prefix=prefix), + ruby, runas=runas).splitlines(): m = re.match("^([^ ]+) \((.+)\)", line) if m: gem = m.group(1) @@ -93,7 +101,8 @@ def list(prefix = "", ruby = None, runas = None): gems[gem] = versions return gems -def sources_add(source_uri, ruby = None, runas = None): + +def sources_add(source_uri, ruby=None, runas=None): """ Add a gem source. @@ -104,9 +113,11 @@ def sources_add(source_uri, ruby = None, runas = None): runas : None The user to run gem as. """ - return _gem("sources --add {source_uri}".format(source_uri = source_uri), ruby, runas = runas) + return _gem("sources --add {source_uri}". + format(source_uri=source_uri), ruby, runas=runas) -def sources_remove(source_uri, ruby = None, runas = None): + +def sources_remove(source_uri, ruby=None, runas=None): """ Remove a gem source. @@ -117,9 +128,11 @@ def sources_remove(source_uri, ruby = None, runas = None): runas : None The user to run gem as. """ - return _gem("sources --remove {source_uri}".format(source_uri = source_uri), ruby, runas = runas) + return _gem("sources --remove {source_uri}". + format(source_uri=source_uri), ruby, runas=runas) + -def sources_list(ruby = None, runas = None): +def sources_list(ruby=None, runas=None): """ List the configured gem sources. @@ -128,4 +141,4 @@ def sources_list(ruby = None, runas = None): runas : None The user to run gem as. """ - return _gem("sources", ruby, runas = runas).splitlines()[2:] + return _gem("sources", ruby, runas=runas).splitlines()[2:] diff --git a/salt/modules/rvm.py b/salt/modules/rvm.py index 737961fa1958..0f2040de7a5d 100644 --- a/salt/modules/rvm.py +++ b/salt/modules/rvm.py @@ -8,18 +8,26 @@ import re -def _rvm(command, arguments = "", runas = None): + +def _rvm(command, arguments="", runas=None): if not is_installed(): return False - ret = __salt__["cmd.run_all"]("/usr/local/rvm/bin/rvm {command} {arguments}".format(command = command, arguments = arguments), runas = runas or __opts__["rvm.runas"]) + ret = __salt__["cmd.run_all"]( + "/usr/local/rvm/bin/rvm {command} {arguments}". + format(command=command, arguments=arguments), + runas=runas or __opts__["rvm.runas"]) if ret["retcode"] == 0: return ret["stdout"] else: return False -def _rvm_do(ruby, command, runas = None): - return _rvm("{ruby} do {command}".format(ruby = ruby or "default", command = command), runas = runas) + +def _rvm_do(ruby, command, runas=None): + return _rvm("{ruby} do {command}". + format(ruby=ruby or "default", command=command), + runas=runas) + def is_installed(): """ @@ -27,22 +35,20 @@ def is_installed(): """ return __salt__["cmd.has_exec"]("/usr/local/rvm/bin/rvm") -# RVM dependencies on Ubuntu 10.04: -# bash coreutils gzip bzip2 gawk sed curl git-core subversion sudo + def install(): """ Install RVM system wide. """ + # RVM dependencies on Ubuntu 10.04: + # bash coreutils gzip bzip2 gawk sed curl git-core subversion sudo + installer = "https://raw.github.com/wayneeseguin/rvm/master/binscripts/rvm-installer" return 0 == __salt__["cmd.retcode"]( # the RVM installer only does a multi-user install when it is invoked under sudo - "curl -s https://raw.github.com/wayneeseguin/rvm/master/binscripts/rvm-installer | sudo bash -s stable") - -# MRI/RBX/REE dependencies for Ubuntu 10.04: -# build-essential openssl libreadline6 libreadline6-dev curl -# git-core zlib1g zlib1g-dev libssl-dev libyaml-dev libsqlite3-0 -# libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev autoconf libc6-dev -# libncurses5-dev automake libtool bison subversion ruby -def install_ruby(ruby, runas = None): + "curl -s {installer} | sudo bash -s stable").format(installer=installer) + + +def install_ruby(ruby, runas=None): """ Install a ruby implementation. @@ -51,9 +57,15 @@ def install_ruby(ruby, runas = None): runas : None The user to run rvm as. """ - return _rvm("install", ruby, runas = runas) + # MRI/RBX/REE dependencies for Ubuntu 10.04: + # build-essential openssl libreadline6 libreadline6-dev curl + # git-core zlib1g zlib1g-dev libssl-dev libyaml-dev libsqlite3-0 + # libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev autoconf libc6-dev + # libncurses5-dev automake libtool bison subversion ruby + return _rvm("install", ruby, runas=runas) + -def reinstall_ruby(ruby, runas = None): +def reinstall_ruby(ruby, runas=None): """ Reinstall a ruby implementation. @@ -62,9 +74,10 @@ def reinstall_ruby(ruby, runas = None): runas : None The user to run rvm as. """ - return _rvm("reinstall", ruby, runas = runas) + return _rvm("reinstall", ruby, runas=runas) -def list(runas = None): + +def list(runas=None): """ List all rvm installed rubies. @@ -72,14 +85,15 @@ def list(runas = None): The user to run rvm as. """ rubies = [] - - for line in _rvm("list", "", runas = runas).splitlines(): + + for line in _rvm("list", "", runas=runas).splitlines(): m = re.match("^[= ]([*> ]) ([^- ]+)-([^ ]+) \[ (.*) \]", line) if m: rubies.append([m.group(2), m.group(3), m.group(1) == "*"]) return rubies -def set_default(ruby, runas = None): + +def set_default(ruby, runas=None): """ Set the default ruby. @@ -88,9 +102,10 @@ def set_default(ruby, runas = None): runas : None The user to run rvm as. """ - return _rvm("alias", "create default {ruby}".format(ruby = ruby), runas = runas) + return _rvm("alias", "create default {ruby}".format(ruby=ruby), runas=runas) -def get(version = "stable", runas = None): + +def get(version="stable", runas=None): """ Update RVM. @@ -99,9 +114,10 @@ def get(version = "stable", runas = None): ruby The version of ruby to reinstall. """ - return _rvm("get", version, runas = runas) + return _rvm("get", version, runas=runas) + -def wrapper(ruby_string, wrapper_prefix, runas = None, *binaries): +def wrapper(ruby_string, wrapper_prefix, runas=None, *binaries): """ Install RVM wrapper scripts. @@ -115,11 +131,12 @@ def wrapper(ruby_string, wrapper_prefix, runas = None, *binaries): The names of the binaries to create wrappers for. When nothing is given, wrappers for ruby, gem, rake, irb, rdoc, ri and testrb are generated. """ return _rvm("wrapper", "{ruby_string} {wrapper_prefix} {binaries}". - format(ruby_string = ruby_string, - wrapper_prefix = wrapper_prefix, - binaries = " ".join(binaries)), runas = runas) + format(ruby_string=ruby_string, + wrapper_prefix=wrapper_prefix, + binaries=" ".join(binaries)), runas=runas) + -def rubygems(ruby, version, runas = None): +def rubygems(ruby, version, runas=None): """ Installs a specific rubygems version in the given ruby. @@ -130,9 +147,10 @@ def rubygems(ruby, version, runas = None): runas : None The user to run rvm as. """ - return _rvm_do(ruby, "rubygems", version, runas = runas) + return _rvm_do(ruby, "rubygems", version, runas=runas) -def gemset_create(ruby, gemset, runas = None): + +def gemset_create(ruby, gemset, runas=None): """ Creates a gemset. @@ -143,28 +161,30 @@ def gemset_create(ruby, gemset, runas = None): runas : None The user to run rvm as. """ - return _rvm_do(ruby, "rvm gemset create {gemset}".format(gemset = gemset), runas = runas) + return _rvm_do(ruby, "rvm gemset create {gemset}".format(gemset=gemset), runas=runas) + -def gemset_list(ruby = "default", runas = None): +def gemset_list(ruby="default", runas=None): """ List all gemsets for the given ruby. - + ruby : default The ruby version to list the gemsets for runas : None The user to run rvm as. """ gemsets = [] - for line in _rvm_do(ruby, "rvm gemset list", runas = runas).splitlines(): + for line in _rvm_do(ruby, "rvm gemset list", runas=runas).splitlines(): m = re.match("^ ([^ ]+)", line) if m: gemsets.append(m.group(1)) return gemsets -def gemset_delete(ruby, gemset, runas = None): + +def gemset_delete(ruby, gemset, runas=None): """ Deletes a gemset. - + ruby The ruby version the gemset belongs to. gemset @@ -172,12 +192,13 @@ def gemset_delete(ruby, gemset, runas = None): runas : None The user to run rvm as. """ - return _rvm_do(ruby, "rvm --force gemset delete {gemset}".format(gemset = gemset), runas = runas) + return _rvm_do(ruby, "rvm --force gemset delete {gemset}".format(gemset=gemset), runas=runas) + -def gemset_empty(ruby, gemset, runas = None): +def gemset_empty(ruby, gemset, runas=None): """ Remove all gems from a gemset. - + ruby The ruby version the gemset belongs to. gemset @@ -185,12 +206,13 @@ def gemset_empty(ruby, gemset, runas = None): runas : None The user to run rvm as. """ - return _rvm_do(ruby, "rvm --force gemset empty", gemset, runas = runas) + return _rvm_do(ruby, "rvm --force gemset empty", gemset, runas=runas) + -def gemset_copy(source, destination, runas = None): +def gemset_copy(source, destination, runas=None): """ Copy all gems from one gemset to another. - + source The name of the gemset to copy, complete with ruby version. destination @@ -198,9 +220,10 @@ def gemset_copy(source, destination, runas = None): runas : None The user to run rvm as. """ - return _rvm("gemset copy", source, destination, runas = runas) + return _rvm("gemset copy", source, destination, runas=runas) -def gemset_list_all(runas = None): + +def gemset_list_all(runas=None): """ List all gemsets for all installed rubies. @@ -210,7 +233,7 @@ def gemset_list_all(runas = None): """ gemsets = {} current_ruby = None - for line in _rvm_do("default", "rvm gemset list_all", runas = runas).splitlines(): + for line in _rvm_do("default", "rvm gemset list_all", runas=runas).splitlines(): m = re.match("^gemsets for ([^ ]+)", line) if m: current_ruby = m.group(1) @@ -220,7 +243,8 @@ def gemset_list_all(runas = None): gemsets[current_ruby].append(m.group(1)) return gemsets -def do(ruby, command, runas = None): + +def do(ruby, command, runas=None): """ Execute a command in an RVM controlled environment. @@ -231,5 +255,4 @@ def do(ruby, command, runas = None): runas : None The user to run rvm as. """ - return _rvm_do(ruby, command, runas = runas) - + return _rvm_do(ruby, command, runas=runas) diff --git a/salt/states/gem.py b/salt/states/gem.py index 422ef8a52f1d..74e2898e17f5 100644 --- a/salt/states/gem.py +++ b/salt/states/gem.py @@ -14,7 +14,8 @@ - ruby: jruby@jgemset """ -def installed(name, ruby = None, runas = None): + +def installed(name, ruby=None, runas=None): """ Make sure that a gem is installed. @@ -26,12 +27,12 @@ def installed(name, ruby = None, runas = None): The user to run gem as. """ ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} - if name in __salt__["gem.list"](name, ruby, runas = runas): + if name in __salt__["gem.list"](name, ruby, runas=runas): ret["result"] = True ret["comment"] = "Gem is already installed." return ret - - if __salt__["gem.install"](name, ruby, runas = runas): + + if __salt__["gem.install"](name, ruby, runas=runas): ret["result"] = True ret["changes"][name] = "Installed" ret["comment"] = "Gem was successfully installed" @@ -41,7 +42,8 @@ def installed(name, ruby = None, runas = None): return ret -def removed(name, ruby = None, runas = None): + +def removed(name, ruby=None, runas=None): """ Make sure that a gem is not installed. @@ -53,12 +55,12 @@ def removed(name, ruby = None, runas = None): The user to run gem as. """ ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} - if name not in __salt__["gem.list"](name, ruby, runas = runas): + if name not in __salt__["gem.list"](name, ruby, runas=runas): ret["result"] = True ret["comment"] = "Gem is not installed." return ret - if __salt__["gem.uninstall"](name, ruby, runas = runas): + if __salt__["gem.uninstall"](name, ruby, runas=runas): ret["result"] = True ret["changes"][name] = "Removed" ret["comment"] = "Gem was successfully removed." @@ -66,5 +68,3 @@ def removed(name, ruby = None, runas = None): ret["result"] = False ret["comment"] = "Could not remove gem." return ret - - diff --git a/salt/states/rvm.py b/salt/states/rvm.py index c1603c51cecb..cc6835503b6c 100644 --- a/salt/states/rvm.py +++ b/salt/states/rvm.py @@ -111,6 +111,7 @@ import re + def _check_rvm(ret): if not __salt__["rvm.is_installed"](): if __salt__["rvm.install"](): @@ -120,10 +121,11 @@ def _check_rvm(ret): ret["comment"] = "Could not install RVM." return ret -def _check_and_install_ruby(ret, ruby, default = False, runas = None): - ret = _check_ruby(ret, ruby, runas = runas) + +def _check_and_install_ruby(ret, ruby, default=False, runas=None): + ret = _check_ruby(ret, ruby, runas=runas) if not ret["result"]: - if __salt__["rvm.install_ruby"](ruby, runas = runas): + if __salt__["rvm.install_ruby"](ruby, runas=runas): ret["result"] = True ret["changes"][ruby] = "Installed" ret["comment"] = "Successfully installed ruby." @@ -134,11 +136,12 @@ def _check_and_install_ruby(ret, ruby, default = False, runas = None): return ret if not ret["default"] and default: - __salt__["rvm.set_default"](ruby, runas = runas) + __salt__["rvm.set_default"](ruby, runas=runas) return ret -def _check_ruby(ret, ruby, runas = None): + +def _check_ruby(ret, ruby, runas=None): match_version = True match_micro_version = False micro_version_regex = re.compile("-([0-9]{4}\.[0-9]{2}|p[0-9]+)$") @@ -147,10 +150,10 @@ def _check_ruby(ret, ruby, runas = None): if re.match("^[a-z]+$", ruby): match_version = False ruby = re.sub("^ruby-", "", ruby) - - for impl, version, default in __salt__["rvm.list"](runas = runas): + + for impl, version, default in __salt__["rvm.list"](runas=runas): if impl != "ruby": - version = "{impl}-{version}".format(impl = impl, version = version) + version = "{impl}-{version}".format(impl=impl, version=version) if not match_micro_version: version = micro_version_regex.sub("", version) if not match_version: @@ -162,9 +165,11 @@ def _check_ruby(ret, ruby, runas = None): break return ret -def installed(name, default = False, runas = None): + +def installed(name, default=False, runas=None): """ - Verify that the specified ruby is installed with RVM. RVM is installed when necessary. + Verify that the specified ruby is installed with RVM. RVM is + installed when necessary. name The version of ruby to install @@ -178,13 +183,14 @@ def installed(name, default = False, runas = None): ret = _check_rvm(ret) if ret["result"] == False: return ret - - return _check_and_install_ruby(ret, name, default, runas = runas) - -def gemset_present(name, ruby = "default", runas = None): + + return _check_and_install_ruby(ret, name, default, runas=runas) + + +def gemset_present(name, ruby="default", runas=None): """ Verify that the gemset is present. - + name The name of the gemset. ruby : default @@ -197,7 +203,7 @@ def gemset_present(name, ruby = "default", runas = None): ret = _check_rvm(ret) if ret["result"] == False: return ret - + if name.find("@") != -1: ruby, name = name.split("@") ret = _check_ruby(ret, ruby) @@ -205,12 +211,12 @@ def gemset_present(name, ruby = "default", runas = None): ret["result"] = False ret["comment"] = "Requested ruby implementation was not found." return ret - - if name in __salt__["rvm.gemset_list"](ruby, runas = runas): + + if name in __salt__["rvm.gemset_list"](ruby, runas=runas): ret["result"] = True ret["comment"] = "Gemset already exists." else: - if __salt__["rvm.gemset_create"](ruby, name, runas = runas): + if __salt__["rvm.gemset_create"](ruby, name, runas=runas): ret["result"] = True ret["comment"] = "Gemset successfully created." ret["changes"][name] = "created" From edf1a439232914810af15e0db6016e133e5ad101 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 23 Feb 2012 14:19:03 -0700 Subject: [PATCH 071/598] chage default user logic to create usable login users by default --- salt/modules/pw_user.py | 13 +++++++++---- salt/modules/useradd.py | 13 +++++++++---- salt/states/user.py | 4 ++-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/salt/modules/pw_user.py b/salt/modules/pw_user.py index 5cd8e7eb70af..0a4fe7c1b98f 100644 --- a/salt/modules/pw_user.py +++ b/salt/modules/pw_user.py @@ -18,8 +18,8 @@ def add(name, uid=None, gid=None, groups=None, - home=False, - shell='/bin/false'): + home=True, + shell=None): ''' Add a user to the minion @@ -29,7 +29,9 @@ def add(name, ''' if isinstance(groups, basestring): groups = groups.split(',') - cmd = 'pw useradd -s {0} '.format(shell) + cmd = 'pw useradd ' + if shell: + cmd += '-s {0} '.format(shell) if uid: cmd += '-u {0} '.format(uid) if gid: @@ -37,7 +39,10 @@ def add(name, if groups: cmd += '-G {0} '.format(','.join(groups)) if home: - cmd += '-m -b {0} '.format(os.dirname(home)) + if home is True: + cmd += '-m ' + else: + cmd += '-m -b {0} '.format(os.dirname(home)) cmd += '-n {0}'.format(name) ret = __salt__['cmd.run_all'](cmd) diff --git a/salt/modules/useradd.py b/salt/modules/useradd.py index 7561548fb551..e00615dd1582 100644 --- a/salt/modules/useradd.py +++ b/salt/modules/useradd.py @@ -17,8 +17,8 @@ def add(name, uid=None, gid=None, groups=None, - home=False, - shell='/bin/false'): + home=True, + shell=None): ''' Add a user to the minion @@ -28,7 +28,9 @@ def add(name, ''' if isinstance(groups, basestring): groups = groups.split(',') - cmd = 'useradd -s {0} '.format(shell) + cmd = 'useradd ' + if shell: + cmd += '-s {0} '.format(shell) if uid: cmd += '-u {0} '.format(uid) if gid: @@ -36,7 +38,10 @@ def add(name, if groups: cmd += '-G {0} '.format(','.join(groups)) if home: - cmd += '-m -d {0} '.format(home) + if home is True: + cmd += '-m ' + else: + cmd += '-m -d {0} '.format(home) cmd += name ret = __salt__['cmd.run_all'](cmd) diff --git a/salt/states/user.py b/salt/states/user.py index c582c95bf4c0..4e8a2640a7cd 100644 --- a/salt/states/user.py +++ b/salt/states/user.py @@ -27,7 +27,7 @@ def present( groups=None, home=False, password=None, - shell='/bin/bash' + shell=None ): ''' Ensure that the named user is present with the specified properties @@ -52,7 +52,7 @@ def present( A password hash to set for the user shell - The login shell, defaults to /bin/bash + The login shell, defaults to the system default shell ''' ret = {'name': name, 'changes': {}, From 00dc99634a0c66c5129dca1583e07b4c85795ba5 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Thu, 23 Feb 2012 23:52:52 -0700 Subject: [PATCH 072/598] Removed password and added other default options to add() This allows for users to be managed by Salt States --- salt/modules/win_useradd.py | 50 +++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_useradd.py b/salt/modules/win_useradd.py index 03b3b1053777..9f3c1a5a15c2 100644 --- a/salt/modules/win_useradd.py +++ b/salt/modules/win_useradd.py @@ -11,7 +11,7 @@ def __virtual__(): return 'user' if __grains__['kernel'] == 'Windows' else False -def add(name, password): +def add(name, uid=None, gid=None, groups=None, home=False, shell=None ): ''' Add a user to the minion @@ -19,7 +19,7 @@ def add(name, password): salt '*' user.add name password ''' - cmd = 'net user {0} {1} /add'.format(name, password) + cmd = 'net user {0} /add'.format(name) ret = __salt__['cmd.run_all'](cmd) return not ret['retcode'] @@ -188,3 +188,49 @@ def list_groups(name): ugrp.add(group) return sorted(list(ugrp)) + + +def getent(): + ''' + Return the list of all info for all users + + CLI Example:: + + salt '*' user.getent + ''' + ret = [] + items = {} + users = [] + startusers = False + cmd = 'net user' + lines = __salt__['cmd.run'](cmd).split('\n') + for line in lines: + if '----------' in line: + startusers = True + continue + if startusers: + if 'successfully' not in line: + comps = line.split() + users += comps + ##if not len(comps) > 1: + #continue + #items[comps[0].strip()] = comps[1].strip() + #return users + for user in users: + stuff = {} + info = __salt__['user.info'](user) + uid = __salt__['file.user_to_uid'](info['name']) + + stuff['gid'] = '' + stuff['groups'] = info['groups'] + stuff['home'] = info['home'] + stuff['name'] = info['name'] + stuff['passwd'] = '' + stuff['shell'] = '' + stuff['uid'] = uid + + ret.append(stuff) + + + return ret + From 0daf69df96612f8e6028564064daa9bff098a937 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Fri, 24 Feb 2012 00:08:50 -0700 Subject: [PATCH 073/598] Add win_shadow.py and some other changes so that Salt States can manage users on Windows --- salt/modules/shadow.py | 13 ++++++++++ salt/modules/win_file.py | 17 +++++++++++++ salt/modules/win_shadow.py | 50 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 salt/modules/win_shadow.py diff --git a/salt/modules/shadow.py b/salt/modules/shadow.py index 318e9425127f..9604728aecda 100644 --- a/salt/modules/shadow.py +++ b/salt/modules/shadow.py @@ -5,6 +5,19 @@ import os import spwd +def __virtual__(): + ''' + Only work on posix-like systems + ''' + + # Disable on these platforms, specific file modules exist: + disable = [ + 'Windows', + ] + if __grains__['os'] in disable: + return False + return 'file' + def info(name): ''' diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py index dc55019b0fa5..8bca791bd616 100644 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -124,6 +124,23 @@ def get_uid(path): owner_sid = secdesc.GetSecurityDescriptorOwner() return win32security.ConvertSidToStringSid(owner_sid) +def get_mode(path): + ''' + Return the mode of a file + + Right now we're just returning 0777 + because Windows' doesn't have a mode + like Linux + + CLI Example:: + + salt '*' file.get_mode /etc/passwd + ''' + if not os.path.exists(path): + return -1 + mode = 0777 + return mode + def get_user(path): ''' diff --git a/salt/modules/win_shadow.py b/salt/modules/win_shadow.py new file mode 100644 index 000000000000..ca29448d6a47 --- /dev/null +++ b/salt/modules/win_shadow.py @@ -0,0 +1,50 @@ +''' +Manage the shadow file +''' + +import os + +def __virtual__(): + ''' + Only works on Windows systems + ''' + if __grains__['os'] == 'Windows': + return 'shadow' + return False + + +def info(name): + ''' + Return information for the specified user + This is just returns dummy data so that it states can work. + + CLI Example:: + + salt '*' shadow.info root + ''' + ret = { + 'name': name, + 'pwd': '', + 'lstchg': '', + 'min': '', + 'max': '', + 'warn': '', + 'inact': '', + 'expire': ''} + return ret + + +def set_password(name, password): + ''' + Set the password for a named user. The password must be a properly defined + hash, the password hash can be generated with this command: + ``openssl passwd -1 `` + + CLI Example:: + + salt '*' shadow.set_password root $1$UYCIxa628.9qXjpQCjM4a.. + ''' + cmd = 'net user {0} {1}'.format(name, password) + ret = __salt__['cmd.run_all'](cmd) + + return not ret['retcode'] From 5379d5da689a2cb2d853812f73757270b180f194 Mon Sep 17 00:00:00 2001 From: Erik Johnson <palehose@gmail.com> Date: Fri, 24 Feb 2012 18:07:40 -0600 Subject: [PATCH 074/598] use hash_file to confirm file's existence before attempting to retrieve it --- salt/modules/cp.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/modules/cp.py b/salt/modules/cp.py index 5a97765207ef..355c9e1f7712 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -43,8 +43,11 @@ def get_file(path, dest, env='base'): salt '*' cp.get_file salt://path/to/file /minion/dest ''' - client = salt.minion.FileClient(__opts__) - return client.get_file(path, dest, False, env) + if not hash_file(path,env): + return '' + else: + client = salt.minion.FileClient(__opts__) + return client.get_file(path, dest, False, env) def get_dir(path, dest, env='base'): From 802b547ff4a03cb885d6e1f0f836808b5d30d71c Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Fri, 24 Feb 2012 22:17:18 -0700 Subject: [PATCH 075/598] Fixed comments and simplified some code per SEJeff's suggestions --- salt/modules/shadow.py | 7 ++----- salt/modules/win_file.py | 4 ++-- salt/modules/win_shadow.py | 8 +++----- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/salt/modules/shadow.py b/salt/modules/shadow.py index 9604728aecda..2d7338a045cc 100644 --- a/salt/modules/shadow.py +++ b/salt/modules/shadow.py @@ -10,11 +10,8 @@ def __virtual__(): Only work on posix-like systems ''' - # Disable on these platforms, specific file modules exist: - disable = [ - 'Windows', - ] - if __grains__['os'] in disable: + # Disable on Windows, a specific file module exists: + if __grains__['os'] == 'Windows': return False return 'file' diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py index 8bca791bd616..dd93171023db 100644 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -128,7 +128,7 @@ def get_mode(path): ''' Return the mode of a file - Right now we're just returning 0777 + Right now we're just returning 777 because Windows' doesn't have a mode like Linux @@ -138,7 +138,7 @@ def get_mode(path): ''' if not os.path.exists(path): return -1 - mode = 0777 + mode = 777 return mode diff --git a/salt/modules/win_shadow.py b/salt/modules/win_shadow.py index ca29448d6a47..a7d956600000 100644 --- a/salt/modules/win_shadow.py +++ b/salt/modules/win_shadow.py @@ -16,7 +16,7 @@ def __virtual__(): def info(name): ''' Return information for the specified user - This is just returns dummy data so that it states can work. + This is just returns dummy data so that salt states can work. CLI Example:: @@ -36,13 +36,11 @@ def info(name): def set_password(name, password): ''' - Set the password for a named user. The password must be a properly defined - hash, the password hash can be generated with this command: - ``openssl passwd -1 <plaintext password>`` + Set the password for a named user. CLI Example:: - salt '*' shadow.set_password root $1$UYCIxa628.9qXjpQCjM4a.. + salt '*' shadow.set_password root mysecretpassword ''' cmd = 'net user {0} {1}'.format(name, password) ret = __salt__['cmd.run_all'](cmd) From 99ec9edd79c1559138ae958549a9c1b790220146 Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Fri, 24 Feb 2012 23:17:06 -0700 Subject: [PATCH 076/598] Move ternary expressions out of format() calls in pip This would break if you didn't specify a virtualenv and a requirements file when calling out to pip. Fixes #754. Thanks for the report @araddon! --- salt/modules/pip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index d5aafa1c03c7..3dddce3acd3d 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -44,8 +44,8 @@ def install(env='', requirements='', pkgs='', pip_bin=''): ''' cmd = '{pip_bin} install {env} {reqs} {pkgs}'.format( pip_bin=_get_pip_bin(pip_bin, env), - env='-E {0}'.format(env if env else ''), - reqs='-r {0}'.format(requirements if requirements else ''), + env='-E {0}'.format(env) if env else '', + reqs='-r {0}'.format(requirements) if requirements else '', pkgs=pkgs) return __salt__['cmd.run'](cmd) From 1dc48fc94053c397ef1ef7af63fefaa79b96fffe Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Fri, 24 Feb 2012 23:32:09 -0700 Subject: [PATCH 077/598] Add ability for Salt States to manage groups on Windows --- salt/modules/win_groupadd.py | 23 ++++++++++++++++++++--- salt/modules/win_useradd.py | 26 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index 9144bf348044..d5a8336e3576 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -9,7 +9,7 @@ def __virtual__(): return 'group' if __grains__['kernel'] == 'Windows' else False -def add(name): +def add(name, gid=None): ''' Add the specified group @@ -77,6 +77,7 @@ def getent(): salt '*' group.getent ''' ret = [] + ret2 = [] lines = __salt__['cmd.run']('net localgroup').split('\n') groupline = False for line in lines: @@ -86,5 +87,21 @@ def getent(): ret.append(line.strip('*').strip()) if '---' in line: groupline = True - - return ret + for item in ret: + members = [] + gid = __salt__['file.group_to_gid'](item) + memberlines = __salt__['cmd.run']('net localgroup "{0}"'.format(item)).split('\n') + memberline = False + for line in memberlines: + if 'successfully' in line: + memberline = False + if memberline: + members.append(line.strip('*').strip()) + if '---' in line: + memberline = True + group = {'gid': gid, + 'members': members, + 'name': item, + 'passwd': 'x'} + ret2.append(group) + return ret2 diff --git a/salt/modules/win_useradd.py b/salt/modules/win_useradd.py index 9f3c1a5a15c2..79db33c44fba 100644 --- a/salt/modules/win_useradd.py +++ b/salt/modules/win_useradd.py @@ -132,6 +132,32 @@ def chprofile(name, profile): return post_info['profile'] == profile return False + +def chgroups(name, groups, append=False): + ''' + Change the groups this user belongs to, add append to append the specified + groups + + CLI Example:: + + salt '*' user.chgroups foo wheel,root True + ''' + if isinstance(groups, basestring): + groups = groups.split(',') + ugrps = set(list_groups(name)) + if ugrps == set(groups): + return True + if not append: + for group in list_groups(name): + cmd = 'net localgroup {0} {1} /delete'.format(group, name) + __salt__['cmd.run'](cmd) + for group in groups: + cmd = 'net localgroup {0} {1} /add'.format(group, name) + __salt__['cmd.run'](cmd) + agrps = set(list_groups(name)) + return len(ugrps - agrps) == 0 + + def info(name): ''' Return user information From 8070d1f47978bc41d0724ccb19526f312de32764 Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Sun, 19 Feb 2012 23:20:41 -0700 Subject: [PATCH 078/598] Made steps 1 & 2 more explicitly tutorials --- doc/index.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index 607421d42bda..53d51b1f2876 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -22,8 +22,8 @@ fast and efficient using **msgpack**; and extensible using small and simple Read the :doc:`Salt overview <topics/index>` for a more thorough description. -Step 1: Remote execution ------------------------- +Tutorial 1: Remote execution +---------------------------- .. sidebar:: |vid| Screencasts @@ -39,8 +39,8 @@ execution` — executing commands on remote hosts. 2. :doc:`Configure the minion <topics/configuration>` 3. :doc:`Run remote commands <topics/tutorials/modules>` -Step 2: Configuration management --------------------------------- +Tutorial 2: Configuration management +------------------------------------ Now that you have the basics out of the way, learn to use Salt to configure your servers. This is widely known as :term:`configuration management` — From 65646830526d7106b999eb2e200b85f398149960 Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Sun, 19 Feb 2012 23:22:24 -0700 Subject: [PATCH 079/598] Updated legacy reference to the minion id as 'hostname' --- doc/ref/configuration/minion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index aff3cc3c19f9..0146c84c5b92 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -69,7 +69,7 @@ The directory used to store the minion's public and private keys. pki_dir: /etc/salt/pki -.. conf_minion:: hostname +.. conf_minion:: id ``id`` ------------ From b72fa96756582efc11c3a1f117999afd60a215c7 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 00:09:30 -0700 Subject: [PATCH 080/598] Add support for matching grains that are in a list Fixes #752, Before this matching grains in a list was difficult at best. Now matching will match any individual member of a list. This makes adding a roles grain much easier! --- salt/minion.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/minion.py b/salt/minion.py index 653781352edd..4db84414d21d 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -599,6 +599,12 @@ def grain_match(self, tgt): if comps[0] not in self.opts['grains']: log.error('Got unknown grain from master: {0}'.format(comps[0])) return False + if isinstance(self.opts['grains'][comps[0], list): + # We are matching a single component to a single list member + for member in self.opts['grains'][comps[0]: + if re.match(comps[1], str(self.opts['grains'][comps[0]])): + return True + return False return bool(re.match(comps[1], str(self.opts['grains'][comps[0]]))) def exsel_match(self, tgt): From 23089ed7a7c997183a0aff812aa263d0ecfd9ab8 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 00:11:11 -0700 Subject: [PATCH 081/598] Fix issues in code from previous commit I might need to get some sleep in... --- salt/minion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 4db84414d21d..d83b0b976da5 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -599,10 +599,10 @@ def grain_match(self, tgt): if comps[0] not in self.opts['grains']: log.error('Got unknown grain from master: {0}'.format(comps[0])) return False - if isinstance(self.opts['grains'][comps[0], list): + if isinstance(self.opts['grains'][comps[0]], list): # We are matching a single component to a single list member for member in self.opts['grains'][comps[0]: - if re.match(comps[1], str(self.opts['grains'][comps[0]])): + if re.match(comps[1], str(member)): return True return False return bool(re.match(comps[1], str(self.opts['grains'][comps[0]]))) From 62df7c2d66d2efe5513dd5e80016fc6be733f69a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 00:21:46 -0700 Subject: [PATCH 082/598] Fix docstrings in the rh_service module --- salt/modules/rh_service.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/modules/rh_service.py b/salt/modules/rh_service.py index 39ceb91e827f..541efb110eb6 100644 --- a/salt/modules/rh_service.py +++ b/salt/modules/rh_service.py @@ -58,7 +58,7 @@ def get_disabled(): CLI Example:: - salt '*' service.get_enabled + salt '*' service.get_disabled ''' rlevel = _runlevel() ret = set() @@ -78,7 +78,7 @@ def get_all(): CLI Example:: - salt '*' service.get_enabled + salt '*' service.get_all ''' return sorted(get_enabled() + get_disabled()) @@ -120,13 +120,12 @@ def restart(name): def status(name, sig=None): ''' - Return the status for a service, returns the PID or an empty string if the - service is running or not, pass a signature to use to find the service via - ps + Return the status for a service, returns a bool whether the service is + running. CLI Example:: - salt '*' service.status <service name> [service signature] + salt '*' service.status <service name> ''' cmd = 'service {0} status'.format(name) return not __salt__['cmd.retcode'](cmd) From 29a9992c5fbc349d81178e7380284cfc5688e662 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 00:29:16 -0700 Subject: [PATCH 083/598] Fix glob matching to use fnmatch --- salt/minion.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index d83b0b976da5..945755a39882 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -5,10 +5,10 @@ # Import python libs import BaseHTTPServer import contextlib -import glob import logging import multiprocessing import hashlib +import fnmatch import os import re import shutil @@ -567,14 +567,7 @@ def glob_match(self, tgt): ''' Returns true if the passed glob matches the id ''' - tmp_dir = tempfile.mkdtemp() - cwd = os.getcwd() - os.chdir(tmp_dir) - open(self.opts['id'], 'w+').write('salt') - ret = bool(glob.glob(tgt)) - os.chdir(cwd) - shutil.rmtree(tmp_dir) - return ret + return fnmatch.fnmatch(tgt, self.opts['id']) def pcre_match(self, tgt): ''' From ff267ce187d2824675813b4d8e8f5cf4041c1574 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 00:32:29 -0700 Subject: [PATCH 084/598] fix ref to dict in grain matching addition --- salt/minion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index 945755a39882..2c3fa5cbdbe6 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -594,7 +594,7 @@ def grain_match(self, tgt): return False if isinstance(self.opts['grains'][comps[0]], list): # We are matching a single component to a single list member - for member in self.opts['grains'][comps[0]: + for member in self.opts['grains'][comps[0]]: if re.match(comps[1], str(member)): return True return False From cd75bdd485501f55b90815c096826aed9130b60b Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 00:38:28 -0700 Subject: [PATCH 085/598] I would rather not talk about it --- salt/minion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index 2c3fa5cbdbe6..de2096c03b71 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -567,7 +567,7 @@ def glob_match(self, tgt): ''' Returns true if the passed glob matches the id ''' - return fnmatch.fnmatch(tgt, self.opts['id']) + return fnmatch.fnmatch(self.opts['id'], tgt) def pcre_match(self, tgt): ''' From 084f8a72595d65085df6ecedc7b4951473aa073d Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Sun, 19 Feb 2012 23:28:45 -0700 Subject: [PATCH 086/598] Added and organized documents on ways of targeting minions --- doc/contents.rst | 1 + doc/index.rst | 29 ++++++--- doc/topics/targeting/compound.rst | 29 +++++++++ doc/topics/targeting/globbing.rst | 81 ++++++++++++++++++++++++ doc/{ref => topics/targeting}/grains.rst | 0 doc/topics/targeting/index.rst | 33 ++++++++++ doc/topics/targeting/nodegroups.rst | 26 ++++++++ 7 files changed, 189 insertions(+), 10 deletions(-) create mode 100644 doc/topics/targeting/compound.rst create mode 100644 doc/topics/targeting/globbing.rst rename doc/{ref => topics/targeting}/grains.rst (100%) create mode 100644 doc/topics/targeting/index.rst create mode 100644 doc/topics/targeting/nodegroups.rst diff --git a/doc/contents.rst b/doc/contents.rst index fc1dce5fbde8..9faa51e772f2 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -8,6 +8,7 @@ Full Table of Contents topics/index topics/configuration + topics/targeting/index topics/tutorials/modules topics/tutorials/states* topics/community diff --git a/doc/index.rst b/doc/index.rst index 53d51b1f2876..ab7db7e4d155 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -66,6 +66,25 @@ truly make it work for you. :local: :depth: 2 +:doc:`Targeting </topics/targeting/index>` +------------------------------------------ + +Targeting is specifying which minions should execute commands or manage server +configuration. + +:doc:`Globbing and regex </topics/targeting/globbing>` + Match minions using globbing and regular expresssions. + +:doc:`Grains </topics/targeting/grains>` + Match minions using bits of static information about the minion such as + OS, software versions, virtualization, CPU, memory, and much more. + +:doc:`Node groups </topics/targeting/nodegroups>` + Statically define groups of minions. + +:doc:`Compound matchers </topics/targeting/compound>` + Combine the above matchers as a single target. + Remote execution ---------------- @@ -84,16 +103,6 @@ arbitrary commands on remote hosts. :doc:`Writing modules <ref/modules/index>` A guide on how to write Salt modules -**Targeting** - Specify which hosts should run commands or manage configuration. - - :doc:`Targeting <ref/targeting/index>` - Hostnames, lists, regular expressions, or define groups. - - :doc:`Grains <ref/grains>` - Bits of static information about a minion such as OS, version, - virtualization, CPU, memory, and much more. - **Returners** Salt returners allow saving minion responses in various datastores or to various locations in addition to display at the CLI. diff --git a/doc/topics/targeting/compound.rst b/doc/topics/targeting/compound.rst new file mode 100644 index 000000000000..acd90c16c692 --- /dev/null +++ b/doc/topics/targeting/compound.rst @@ -0,0 +1,29 @@ +================= +Compound matchers +================= + +.. glossary:: + + Compound matcher + A combination of many target definitions that can be combined with + boolean operators. + +Compound matchers allow very granular minion targeting using any of the +previously discussed matchers. The default matcher is a glob, as usual. If +something other than a glob is used preface it with the letter denoting the +type. Matchers can be joined using boolean ``and``, ``or``, and ``not`` +operators. + +For example, the following command matches all minions that have a hostname +that begins with "webserv" and that are running Debian or it matches any +minions that have a hostname that matches the regular expression +``web-dc1-srv.*``:: + + salt -C 'webserv* and G@os:Debian or E@web-dc1-srv.*' test.ping + +That same example expressed in a :term:`top file` looks like the following:: + + base: + 'webserv* and G@os:Debian or E@web-dc1-srv.*': + - match: compound + - webserver diff --git a/doc/topics/targeting/globbing.rst b/doc/topics/targeting/globbing.rst new file mode 100644 index 000000000000..3a9e290526ea --- /dev/null +++ b/doc/topics/targeting/globbing.rst @@ -0,0 +1,81 @@ +========================== +Matching the ``minion id`` +========================== + +.. glossary:: + + minion id + A unique identifier for a given minion. By default the minion id is the + FQDN of that host but this can be overridden. + +Each minion needs a unique identifier. By default when a minion starts for the +first time it chooses its :abbr:`FQDN (fully qualified domain name)` as that +identifier. The minion id can be overridden via the minion's :conf_minion:`id` +configuration setting. + +.. tip:: minion id and minion keys + + The :term:`minion id` is used to generate the minion's public/private keys + and if it ever changes the master must then accept the new key as though + the minion was a new host. + +Globbing +======== + +The default matching that Salt utilizes is shell-style globbing around the +:term:`minion id`. This also works for states in the :term:`top file`. + +.. note:: + + You must wrap :command:`salt` calls that use globbing in single-quotes to + prevent the shell from expanding the globs before Salt is invoked. + +Match all minions:: + + salt '*' test.ping + +Match all minions in the example.net domain or any of the example domains:: + + salt '*.example.net' test.ping + salt '*.example.*' test.ping + +Match all the ``webN`` minions in the example.net domain +(``web1.example.net``, ``web2.example.net`` … ``webN.example.net``):: + + salt 'web?.example.net test.ping + +Match the ``web1`` through ``web5`` minions:: + + salt 'web[1-5]' test.ping + +Match the ``web-x``, ``web-y``, and ``web-z`` minions:: + + salt 'web-[x-z]' test.ping + +Regex +===== + +Minions can be matched using Perl-compatible regular expressions (which is +globbing on steroids and a ton of caffeine). + +Match both ``web1-prod`` and ``web1-devel`` minions:: + + salt -E 'web1-(prod|devel)' test.ping + +When using regex in a states :term:`top file` you must specify the matcher as +the first option. The following example executes the contents of +``webserver.sls`` on the above-mentioned minions. + +.. code-block:: yaml + + base: + 'web1-(prod|devel)': + - match: pcre + - webserver + +Lists +===== + +At the most basic you can specify a flat list of minion ids:: + + salt -L 'web1,web2,web3' test.ping diff --git a/doc/ref/grains.rst b/doc/topics/targeting/grains.rst similarity index 100% rename from doc/ref/grains.rst rename to doc/topics/targeting/grains.rst diff --git a/doc/topics/targeting/index.rst b/doc/topics/targeting/index.rst new file mode 100644 index 000000000000..1e22ad484a14 --- /dev/null +++ b/doc/topics/targeting/index.rst @@ -0,0 +1,33 @@ +========= +Targeting +========= + +.. glossary:: + + Targeting + Specifying which minions should run a command or execute a state by + matching against hostnames, or system information, or defined groups, + or even combinations thereof. + +For example the command ``salt web1 apache.signal restart`` to restart the +Apache httpd server specifies the machine ``web1`` as the target and the +command will only be run on that one minion. + +Similarly when using states, the following :term:`top file` specifies that only +the ``web1`` minion should execute the contents of ``webserver.sls``: + +.. code-block:: yaml + + base: + 'web1': + - webserver + +There are many ways to target individual minions or groups of minions in Salt: + +.. toctree:: + :maxdepth: 2 + + globbing + grains + nodegroups + compound diff --git a/doc/topics/targeting/nodegroups.rst b/doc/topics/targeting/nodegroups.rst new file mode 100644 index 000000000000..c15bae84bfb2 --- /dev/null +++ b/doc/topics/targeting/nodegroups.rst @@ -0,0 +1,26 @@ +=========== +Node groups +=========== + +.. glossary:: + + Node group + A predefined group of minions declared in the master configuration file + :conf_master:`nodegroups` setting as a :term:`compound target`. + +For example, in the master config file :conf_master:`nodegroups` setting:: + + nodegroups: + group1: 'L@foo.domain.com,bar.domain.com,baz.domain.com and bl*.domain.com' + group2: 'G@os:Debian and foo.domain.com' + +Specify a nodegroup via the ``-N`` option at the command-line:: + + salt -N group1 test.ping + +Specify a nodegroup with ``- match: nodegroup`` in a :term:`top file`:: + + base: + group1: + - match: nodegroup + - webserver From 738ec9b02911f55dbc101c2d121ca1aab733e6b5 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 01:35:09 -0700 Subject: [PATCH 087/598] change redirect to provider Jeff wins, redirect is not provider (I am still open to suggestions since I would like something else, but this is making the most sense so far) --- salt/state.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/salt/state.py b/salt/state.py index 955dee8c3367..6de2d2cbebeb 100644 --- a/salt/state.py +++ b/salt/state.py @@ -130,16 +130,16 @@ def load_modules(self, data=None): log.info('Loading fresh modules for state activity') self.functions = salt.loader.minion_mods(self.opts) if isinstance(data, dict): - if data.get('redirect', False): - redirect = {} - if isinstance(data['redirect'], str): - redirects = [{data['state']: data['redirect']}] - elif isinstance(data['redirect'], list): - redirects = data['redirect'] - for redirect in redirects: - for mod in redirect: + if data.get('provider', False): + provider = {} + if isinstance(data['provider'], str): + providers = [{data['state']: data['provider']}] + elif isinstance(data['provider'], list): + providers = data['provider'] + for provider in providers: + for mod in provider: funcs = salt.loader.raw_mod(self.opts, - redirect[mod], + provider[mod], self.functions) if funcs: for func in funcs: @@ -570,14 +570,14 @@ def call(self, data): data ) ) - if 'redirect' in data: + if 'provider' in data: self.load_modules(data) cdata = self.format_call(data) ret = self.states[cdata['full']](*cdata['args']) ret['__run_num__'] = self.__run_num self.__run_num += 1 format_log(ret) - if 'redirect' in data: + if 'provider' in data: self.load_modules() return ret From 61edeecdf69e79a0fa4246b5ef9ceba78aaa746a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 01:52:48 -0700 Subject: [PATCH 088/598] Add providers doc --- doc/ref/states/providers.rst | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 doc/ref/states/providers.rst diff --git a/doc/ref/states/providers.rst b/doc/ref/states/providers.rst new file mode 100644 index 000000000000..ee32563e1bc1 --- /dev/null +++ b/doc/ref/states/providers.rst @@ -0,0 +1,47 @@ +=============== +State Providers +=============== + +.. versionadded:: 0.9.8 + +Salt predetermines what modules should be mapped to what uses based on the +properties of a system. These determinations are generally made for modules +that provide things like package and service management. + +Sometimes in states it may be needed for an alternative module to be used +to provide the functionality needed. For instance, an Arch Linux system may +have been set up with systemd support, so instead of using the default service +module detected for Arch Linux, the systemd module can be used: + +.. code-block:: yaml + + httpd: + service: + - running + - enable: True + - provider: systemd + +In this instance the systemd module will replace the service virtual module +which is used by default on Arch Linux, and the httpd service will be set up +using systemd. + +Arbitrary Module Redirects +=========================== + +The provider statement can also be used for more powerful means, instead of +overwriting or extending the module used for the named service an arbitrary +module can be used to provide certain functionality. + +.. code-block:: yaml + + emacs: + pkg: + - installed + - provider: + - pkg: yumpkg5 + - cmd: customcmd + +In this example the default pkg module is being redirected to use the yumpkg5 +module (yum via shelling out instead of via the yum api), but is also using +a custom module to invoke commands. This could be used to dramatically change +the behavior of a given state. From d9b17cede24923a20fba581b6af15fc5e109242b Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Sat, 25 Feb 2012 14:15:11 -0700 Subject: [PATCH 089/598] Re-added the installation docs Since RTD build the latest docs as the default available docs having the installation instructions along with the rest of the docs is really the best place for them to live. Fixes #762 Fixes #763 --- doc/contents.rst | 1 + doc/index.rst | 2 +- doc/topics/installation/arch.rst | 8 +++++ doc/topics/installation/debian.rst | 34 +++++++++++++++++++++ doc/topics/installation/fedora.rst | 26 ++++++++++++++++ doc/topics/installation/freebsd.rst | 7 +++++ doc/topics/installation/gentoo.rst | 18 +++++++++++ doc/topics/installation/index.rst | 47 +++++++++++++++++++++++++++++ 8 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 doc/topics/installation/arch.rst create mode 100644 doc/topics/installation/debian.rst create mode 100644 doc/topics/installation/fedora.rst create mode 100644 doc/topics/installation/freebsd.rst create mode 100644 doc/topics/installation/gentoo.rst create mode 100644 doc/topics/installation/index.rst diff --git a/doc/contents.rst b/doc/contents.rst index 9faa51e772f2..666331a0d579 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -7,6 +7,7 @@ Full Table of Contents :glob: topics/index + topics/installation/index topics/configuration topics/targeting/index topics/tutorials/modules diff --git a/doc/index.rst b/doc/index.rst index ab7db7e4d155..b3c0fffa7468 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -35,7 +35,7 @@ The quickest way to see Salt in action is to run a command on a :term:`minion` host from the :term:`master` host. This is widely known as :term:`remote execution` — executing commands on remote hosts. -1. `Installation`_ +1. :doc:`Installation </topics/installation/index>` 2. :doc:`Configure the minion <topics/configuration>` 3. :doc:`Run remote commands <topics/tutorials/modules>` diff --git a/doc/topics/installation/arch.rst b/doc/topics/installation/arch.rst new file mode 100644 index 000000000000..f6da483f2c45 --- /dev/null +++ b/doc/topics/installation/arch.rst @@ -0,0 +1,8 @@ +========== +Arch Linux +========== + +Salt can be easily installed from the Arch Linux AUR in one of two flavors: + +* `Install a Salt release <https://aur.archlinux.org/packages.php?ID=47512>`_ +* `Install the latest Salt from Git <https://aur.archlinux.org/packages.php?ID=47513>`_ diff --git a/doc/topics/installation/debian.rst b/doc/topics/installation/debian.rst new file mode 100644 index 000000000000..efc9caccc189 --- /dev/null +++ b/doc/topics/installation/debian.rst @@ -0,0 +1,34 @@ +=============== +Debian & Ubuntu +=============== + +Ubuntu +------ + +We are working to get Salt into apt. In the meantime we have a PPA available +for Lucid:: + + aptitude -y install python-software-properties + add-apt-repository ppa:saltstack/salt + aptitude update + aptitude install salt + +Debian +------ + +`A deb package is currently in testing`__ for inclusion in apt. Until that is +accepted you can install Salt by downloading the latest ``.deb`` in the +`downloads section on GitHub`__ and installing that manually using ``dpkg -i``. + +.. __: http://mentors.debian.net/package/salt +.. __: https://github.com/saltstack/salt/downloads + +.. admonition:: Installing ZeroMQ on Squeeze (Debian 6) + + There is a `python-zmq`__ package available in Debian \"wheezy (testing)\". + If you don't have that repo enabled the best way to install Salt and pyzmq + is by using ``pip`` (or ``easy_install``):: + + pip install pyzmq salt + +.. __: http://packages.debian.org/search?keywords=python-zmq diff --git a/doc/topics/installation/fedora.rst b/doc/topics/installation/fedora.rst new file mode 100644 index 000000000000..ce95b03d2f31 --- /dev/null +++ b/doc/topics/installation/fedora.rst @@ -0,0 +1,26 @@ +================================== +Fedora & Enterprise Linux / CentOS +================================== + +We are working to get Salt packages into EPEL. In the meantime you can ``yum +install salt-master salt-minion`` via our Fedora People repository. + +Red Hat Enterprise Linux 5 & 6 or CentOS 5 & 6 +---------------------------------------------- + +1. Install the `EPEL`__ repository. + +2. Install our repository on FedoraPeople:: + + wget -O /etc/yum.repos.d/epel-salt.repo \\ + http://repos.fedorapeople.org/repos/herlo/salt/epel-salt.repo + +.. __: http://fedoraproject.org/wiki/EPEL#How_can_I_use_these_extra_packages.3F + +Fedora 15 & 16 +-------------- + +1. Install our repository on FedoraPeople:: + + wget -O /etc/yum.repos.d/fedora-salt.repo \\ + http://repos.fedorapeople.org/repos/herlo/salt/fedora-salt.repo diff --git a/doc/topics/installation/freebsd.rst b/doc/topics/installation/freebsd.rst new file mode 100644 index 000000000000..440ecef32086 --- /dev/null +++ b/doc/topics/installation/freebsd.rst @@ -0,0 +1,7 @@ +======= +FreeBSD +======= + +Salt is available in the FreeBSD ports tree:: + + cd /usr/ports/sysutils/salt && make install clean diff --git a/doc/topics/installation/gentoo.rst b/doc/topics/installation/gentoo.rst new file mode 100644 index 000000000000..7cf8b8971efd --- /dev/null +++ b/doc/topics/installation/gentoo.rst @@ -0,0 +1,18 @@ +====== +Gentoo +====== + +Salt can be easily installed on Gentoo:: + + emerge pyyaml m2crypto pycrypto jinja pyzmq + +Then download and install from source: + +1. Download the latest source tarball from the GitHub downloads directory for + the Salt project: + +2. Untar the tarball and run the ``setup.py`` as root:: + + tar xvf salt-<version>.tar.gz + cd salt-<version> + python2 setup.py install diff --git a/doc/topics/installation/index.rst b/doc/topics/installation/index.rst new file mode 100644 index 000000000000..217bd75970d6 --- /dev/null +++ b/doc/topics/installation/index.rst @@ -0,0 +1,47 @@ +============ +Installation +============ + +The Salt system setup is amazingly simple, as this is one of the central design +goals of Salt. + +Dependencies +------------ + +Salt should run on any Unix-like platform so long as the dependencies are met. + +* `Python 2.6`_ +* `ZeroMQ`_ >= 2.1.9 +* `pyzmq`_ >= 2.1.9 - ZeroMQ Python bindings +* `M2Crypto`_ - Python OpenSSL wrapper +* `PyCrypto`_ - The Python cryptography toolkit +* `msgpack-python`_ - High-performance message interchange format +* `YAML`_ - Python YAML bindings + +Optional Dependencies +--------------------- + +* `Jinja2`_ - parsing Salt States (configurable in the master settings) +* gcc - dynamic `Cython`_ module compiling + +.. _`Python 2.6`: http://python.org/download/ +.. _`ZeroMQ`: http://www.zeromq.org/ +.. _`pyzmq`: https://github.com/zeromq/pyzmq +.. _`M2Crypto`: http://chandlerproject.org/Projects/MeTooCrypto +.. _`msgpack-python`: http://pypi.python.org/pypi/msgpack-python/0.1.12 +.. _`YAML`: http://pyyaml.org/ +.. _`PyCrypto`: http://www.dlitz.net/software/pycrypto/ +.. _`Cython`: http://cython.org/ +.. _`Jinja2`: http://jinja.pocoo.org/ + +Platform-specific instllation instructions +------------------------------------------ + +.. toctree:: + :maxdepth: 1 + + arch + debian + fedora + freebsd + gentoo From 2ed2241ff777256a70d233eb92431d2182adc4d1 Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Sat, 25 Feb 2012 14:41:27 -0700 Subject: [PATCH 090/598] Moved platform-specific tutorials into the installation instructions --- doc/topics/installation/arch.rst | 262 +++++++++++++++++++++++++++- doc/topics/installation/fedora.rst | 200 +++++++++++++++++++-- doc/topics/installation/freebsd.rst | 214 ++++++++++++++++++++++- doc/topics/tutorials/archlinux.rst | 258 --------------------------- doc/topics/tutorials/fedora.rst | 189 -------------------- doc/topics/tutorials/freebsd.rst | 211 ---------------------- 6 files changed, 651 insertions(+), 683 deletions(-) delete mode 100644 doc/topics/tutorials/archlinux.rst delete mode 100644 doc/topics/tutorials/fedora.rst delete mode 100644 doc/topics/tutorials/freebsd.rst diff --git a/doc/topics/installation/arch.rst b/doc/topics/installation/arch.rst index f6da483f2c45..7af629fccd0e 100644 --- a/doc/topics/installation/arch.rst +++ b/doc/topics/installation/arch.rst @@ -1,8 +1,258 @@ -========== -Arch Linux -========== +.. _introduction: -Salt can be easily installed from the Arch Linux AUR in one of two flavors: +Introduction +============ -* `Install a Salt release <https://aur.archlinux.org/packages.php?ID=47512>`_ -* `Install the latest Salt from Git <https://aur.archlinux.org/packages.php?ID=47513>`_ +Salt has primarily been developed on Arch Linux, meaning it is known to work +very well on that distribution. The lead developer, Thomas S. Hatch (thatch45) has +been a TU (Trusted User) for the Arch Linux distribution, and has written a +number of Arch-specific tools in the past. + +Salt, while not Arch-specific, is packaged for and works well on Arch Linux. + +.. _installation: + +Installation +============ + +Salt is currently available via the Arch User Repository (AUR). There are +currently stable and -git packages available. + +Stable Release +-------------- + +To install Salt stable releases from the Arch Linux AUR, use the commands:: + + wget https://aur.archlinux.org/packages/sa/salt/salt.tar.gz + tar xf salt.tar.gz + cd salt/ + makepkg -is + +A few of Salt's dependencies are currently only found within the AUR, so you'll +need to download and run ``makepkg -is`` on these as well. As a reference, Salt +currently relies on the following packages only available via the AUR: + +* https://aur.archlinux.org/packages/py/python2-msgpack/python2-msgpack.tar.gz +* https://aur.archlinux.org/packages/py/python2-psutil/python2-psutil.tar.gz + +.. note:: yaourt + + If you chose to use a tool such as Yaourt_ the dependencies will be + gathered and built for you automatically. + + The command to install salt using the yaourt tool is: + + .. code-block:: none + + yaourt salt + +.. _Yaourt: https://aur.archlinux.org/packages.php?ID=5863 + +Tracking develop +---------------- + +To install the bleeding edge version of Salt (**may include bugs!**), you can use +the -git package. Installing the -git package can be done using the commands:: + + wget https://aur.archlinux.org/packages/sa/salt-git/salt-git.tar.gz + tar xf salt-git.tar.gz + cd salt-git/ + makepkg -is + +A few of Salt's dependencies are currently only found within the AUR, so you'll +need to download and run ``makepkg -is`` on these as well. As a reference, Salt +currently relies on the following packages only available via the AUR: + +* https://aur.archlinux.org/packages/py/python2-msgpack/python2-msgpack.tar.gz +* https://aur.archlinux.org/packages/py/python2-psutil/python2-psutil.tar.gz + +.. note:: yaourt + + If you chose to use a tool such as Yaourt_ the dependencies will be + gathered and built for you automatically. + + The command to install salt using the yaourt tool is: + + .. code-block:: none + + yaourt salt-git + +.. _Yaourt: https://aur.archlinux.org/packages.php?ID=5863 + +.. _configuration: + +Configuration +============= + +In the sections below I'll outline configuration options for both the Salt +Master and Salt Minions. + +The Salt package installs two template configuration files, /etc/salt/master.template and +/etc/salt/minion.template. You'll need to copy these .template files into place and +make a few edits. First, copy them into place as seen here:: + + cp /etc/salt/master.template /etc/salt/master + cp /etc/salt/minion.template /etc/salt/minion + +Note: You'll only need to copy the config for the service you're going to run. + +Once you've copied the config into place you'll need to make changes specific +to your setup. Below I'll outline suggested configuration changes to the +Master, after which I'll outline configuring the Minion. + +.. _master_configuration: + +Master Configuration +==================== + +This section outlines configuration of a Salt Master, which is used to control +other machines known as "minions" (see "Minion Configuration" for instructions +on configuring a minion). This will outline IP configuration, and a few key +configuration paths. + +**Interface** + +By default the Salt master listens on ports 4505 and 4506 on all interfaces +(0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the +"interface" directive as seen here:: + + - #interface: 0.0.0.0 + + interface: 10.0.0.1 + +**rc.conf** + +Last but not least you'll need to activate the Salt Master in your rc.conf +file. Using your favorite editor, open /etc/rc.conf and add the salt-master:: + + -DAEMONS=(syslog-ng network crond) + +DAEMONS=(syslog-ng network crond @salt-master) + +Once you've completed all of these steps you're ready to start your Salt +Master. You should be able to start your Salt Master now using the command +seen here:: + + rc.d start salt-master + +If your Salt Master doesn't start successfully, go back through each step and +see if anything was missed. Salt doesn't take much configuration (part of its +beauty!), and errors are usually simple mistakes. + +.. _ minion_configuration: + +Minion Configuration +==================== + +Configuring a Salt Minion is surprisingly simple. Unless you have a real need +for customizing your minion configuration (which there are plenty of options if +you are so inclined!), there is one simple directive that needs to be updated. +That option is the location of the master. + +By default a Salt Minion will try to connect to the dns name "salt". If you +have the ability to update DNS records for your domain you might create an A or +CNAME record for "salt" that points to your Salt Master. If you are able to do +this you likely can do without any minion configuration at all. + +If you are not able to update DNS, you'll simply need to update one entry in +the configuration file. Using your favorite editor, open the minion +configuration file and update the "master" entry as seen here:: + + - #master: salt + + master: 10.0.0.1 + +Simply update the master directive to the IP or hostname of your Salt Master. +Save your changes and you're ready to start your Salt Minion. Advanced +configuration options are covered in another chapter. + +**rc.conf** + +Before you're able to start the Salt Minion you'll need to update your rc.conf +file. Using your favorite editor open /etc/rc.conf or /etc/rc.conf.local and +add this line:: + + -DAEMONS=(syslog-ng network crond) + +DAEMONS=(syslog-ng network crond @salt-minion) + +Once you've completed all of these steps you're ready to start your Salt +Minion. You should be able to start your Salt Minion now using the command +seen here:: + + rc.d start salt-minion + +If your Salt Minion doesn't start successfully, go back through each step and +see if anything was missed. Salt doesn't take much configuration (part of its +beauty!), and errors are usually simple mistakes. + +.. _tying_it_all_together: + +Tying It All Together +====================== + +If you've successfully completed each of the steps above you should have a +running Salt Master and a running Salt Minion. The Minion should be configured +to point to the Master. To verify that there is communication flowing between +the Minion and Master we'll run a few initial ``salt`` commands. These commands +will validate the Minions RSA encryption key, and then send a test command to +the Minion to ensure that commands and responses are flowing as expected. + +**Key Management** + +Salt uses AES encryption for all communication between the Master and the +Minion. This ensures that the commands you send to your Minions (your cloud) +can not be tampered with, and that communication between Master and Minion is +only done through trusted, accepted keys. + +Before you'll be able to do any remote execution or configuration management you'll +need to accept any pending keys on the Master. Run the ``salt-key`` command to +list the keys known to the Salt Master:: + + [root@master ~]# salt-key -L + Unaccepted Keys: + avon + bodie + bubbles + marlo + Accepted Keys: + +This example shows that the Salt Master is aware of four Minions, but none of +the keys have been accepted. To accept the keys and allow the Minions to be +controlled by the Master, again use the ``salt-key`` command:: + + [root@master ~]# salt-key -A + [root@master ~]# salt-key -L + Unaccepted Keys: + Accepted Keys: + avon + bodie + bubbles + marlo + +The ``salt-key`` command allows for signing keys individually or in bulk. The +example above, using ``-A`` bulk-accepts all pending keys. To accept keys +individually use the lowercase of the same option, ``-a keyname``. + +.. _sending_commands: + +Sending Commands +================ + +Everything should be set for you to begin remote management of your Minions. +Whether you have a few or a few-dozen, Salt can help you manage them easily! + +For final verification, send a test function from your Salt Master to your +minions. If all of your minions are properly communicating with your Master, +you should "True" responses from each of them. See the example below to send +the ``test.ping`` remote command:: + + [root@master ~]# salt '*' test.ping + {'avon': True} + +.. _where_do_i_go_from_here: + +Where Do I Go From Here +======================== + +Congratulations! You've successfully configured your first Salt Minions and are +able to send remote commands. I'm sure you're eager to learn more about what +Salt can do. Depending on the primary way you want to manage your machines you +may either want to visit the section regarding Salt States, or the section on +Modules. diff --git a/doc/topics/installation/fedora.rst b/doc/topics/installation/fedora.rst index ce95b03d2f31..186514e57637 100644 --- a/doc/topics/installation/fedora.rst +++ b/doc/topics/installation/fedora.rst @@ -1,26 +1,198 @@ -================================== -Fedora & Enterprise Linux / CentOS -================================== +.. _introduction: -We are working to get Salt packages into EPEL. In the meantime you can ``yum -install salt-master salt-minion`` via our Fedora People repository. +Introduction +============ -Red Hat Enterprise Linux 5 & 6 or CentOS 5 & 6 ----------------------------------------------- +Beginning with version 0.9.4, Salt has been available in the primary Fedora +repositories and is available for installation using yum. Fedora will have more +up to date versions of Salt than other members of the Red Hat family, which +makes it a great place to help improve Salt! -1. Install the `EPEL`__ repository. +.. admonition:: CentOS/EL 5 -2. Install our repository on FedoraPeople:: + We are still working to get msgpack packaged for Python 2.6 so Salt is not + yet (but nearly) in EPEL 5. In the meantime you can install Salt via our + Fedora People repository:: wget -O /etc/yum.repos.d/epel-salt.repo \\ http://repos.fedorapeople.org/repos/herlo/salt/epel-salt.repo -.. __: http://fedoraproject.org/wiki/EPEL#How_can_I_use_these_extra_packages.3F +.. _installation: -Fedora 15 & 16 +Installation +============ + +Salt can be installed using ``yum`` and is available in the standard Fedora +repositories. + +Stable Release -------------- -1. Install our repository on FedoraPeople:: +Salt is packaged separately for the minion and the master. You'll only need to +install the appropriate package for the role you need the machine to play. This +means you're going to want one master and a whole bunch of minions!:: + + yum install salt-master + yum install salt-minion + +.. _configuration: + +Configuration +============= + +In the sections below I'll outline configuration options for both the Salt +Master and Salt Minions. + +.. _master_configuration: + +Master Configuration +==================== + +This section outlines configuration of a Salt Master, which is used to control +other machines known as "minions" (see "Minion Configuration" for instructions +on configuring a minion). This will outline IP configuration, and a few key +configuration paths. + +**Interface** + +By default the Salt master listens on ports 4505 and 4506 on all interfaces +(0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the +"interface" directive as seen here:: + + - #interface: 0.0.0.0 + + interface: 10.0.0.1 + +**Enable the Master** + +You'll also likely want to activate the Salt Master in systemd, configuring the +Salt Master to start automatically at boot.:: + + systemctl enable salt-master.service + +Once you've completed all of these steps you're ready to start your Salt +Master. You should be able to start your Salt Master now using the command +seen here:: + + systemctl start salt-master.service + +If your Salt Master doesn't start successfully, go back through each step and +see if anything was missed. Salt doesn't take much configuration (part of its +beauty!), and errors are usually simple mistakes. + +.. _ minion_configuration: + +Minion Configuration +==================== + +Configuring a Salt Minion is surprisingly simple. Unless you have a real need +for customizing your minion configuration (which there are plenty of options if +you are so inclined!), there is one simple directive that needs to be updated. +That option is the location of the master. + +By default a Salt Minion will try to connect to the dns name "salt". If you +have the ability to update DNS records for your domain you might create an A or +CNAME record for "salt" that points to your Salt Master. If you are able to do +this you likely can do without any minion configuration at all. + +If you are not able to update DNS, you'll simply need to update one entry in +the configuration file. Using your favorite editor, open the minion +configuration file and update the "master" entry as seen here:: + + - #master: salt + + master: 10.0.0.1 + +Simply update the master directive to the IP or hostname of your Salt Master. +Save your changes and you're ready to start your Salt Minion. Advanced +configuration options are covered in another chapter. + +**Enable the Minion** + +You'll need to configure the minion to auto-start at boot. You can toggle +that option through systemd.:: + + systemctl enable salt-minion.service + +Once you've completed all of these steps you're ready to start your Salt +Minion. You should be able to start your Salt Minion now using the command +here:: + + systemctl start salt-minion.service + +If your Salt Minion doesn't start successfully, go back through each step and +see if anything was missed. Salt doesn't take much configuration (part of its +beauty!), and errors are usually simple mistakes. + +.. _tying_it_all_together: + +Tying It All Together +====================== + +If you've successfully completed each of the steps above you should have a +running Salt Master and a running Salt Minion. The Minion should be configured +to point to the Master. To verify that there is communication flowing between +the Minion and Master we'll run a few initial ``salt`` commands. These commands +will validate the Minions RSA encryption key, and then send a test command to +the Minion to ensure that commands and responses are flowing as expected. + +**Key Management** + +Salt uses AES encryption for all communication between the Master and the +Minion. This ensures that the commands you send to your Minions (your cloud) +can not be tampered with, and that communication between Master and Minion is +only done through trusted, accepted keys. + +Before you'll be able to do any remote execution or configuration management you'll +need to accept any pending keys on the Master. Run the ``salt-key`` command to +list the keys known to the Salt Master:: + + [root@master ~]# salt-key -L + Unaccepted Keys: + avon + bodie + bubbles + marlo + Accepted Keys: + +This example shows that the Salt Master is aware of four Minions, but none of +the keys have been accepted. To accept the keys and allow the Minions to be +controlled by the Master, again use the ``salt-key`` command:: + + [root@master ~]# salt-key -A + [root@master ~]# salt-key -L + Unaccepted Keys: + Accepted Keys: + avon + bodie + bubbles + marlo + +The ``salt-key`` command allows for signing keys individually or in bulk. The +example above, using ``-A`` bulk-accepts all pending keys. To accept keys +individually use the lowercase of the same option, ``-a keyname``. + +.. _sending_commands: + +Sending Commands +================ + +Everything should be set for you to begin remote management of your Minions. +Whether you have a few or a few-dozen, Salt can help you manage them easily! + +For final verification, send a test function from your Salt Master to your +minions. If all of your minions are properly communicating with your Master, +you should "True" responses from each of them. See the example below to send +the ``test.ping`` remote command:: + + [root@master ~]# salt '*' test.ping + {'avon': True} + +.. _where_do_i_go_from_here: + +Where Do I Go From Here +======================== - wget -O /etc/yum.repos.d/fedora-salt.repo \\ - http://repos.fedorapeople.org/repos/herlo/salt/fedora-salt.repo +Congratulations! You've successfully configured your first Salt Minions and are +able to send remote commands. I'm sure you're eager to learn more about what +Salt can do. Depending on the primary way you want to manage your machines you +may either want to visit the section regarding Salt States, or the section on +Modules. diff --git a/doc/topics/installation/freebsd.rst b/doc/topics/installation/freebsd.rst index 440ecef32086..6eb02ecf46cf 100644 --- a/doc/topics/installation/freebsd.rst +++ b/doc/topics/installation/freebsd.rst @@ -1,7 +1,211 @@ -======= -FreeBSD -======= +.. _introduction: -Salt is available in the FreeBSD ports tree:: +Introduction +============ - cd /usr/ports/sysutils/salt && make install clean +Salt was added to the FreeBSD ports tree Dec 26th, 2011 by Christer Edwards +<christer.edwards@gmail.com>. It has been tested on FreeBSD 8.2 and 9.0 +releases. + +Salt is dependent on the following additional ports. These will be installed as +dependencies of the ``sysutils/salt`` port:: + + /devel/py-yaml + /devel/py-pyzmq + /devel/py-Jinja2 + /devel/py-msgpack + /security/py-pycrypto + /security/py-m2crypto + +.. _installation: + +Installation +============ + +To install Salt from the FreeBSD ports tree, use the command:: + + cd /usr/ports/sysutils/salt && make install clean + +Once the port is installed you'll need to make a few configuration changes. +These include defining the IP to bind to (optional), and some configuration +path changes to make salt fit more natively into the FreeBSD filesystem tree. + +.. _configuration: + +Configuration +============= + +In the sections below I'll outline configuration options for both the Salt +Master and Salt Minions. + +The Salt port installs two sample configuration files, salt/master.sample and +salt/minion.sample (these should be installed in /usr/local/etc/, unless you use a +different %%PREFIX%%). You'll need to copy these .sample files into place and +make a few edits. First, copy them into place as seen here:: + + cp /usr/local/etc/salt/master.sample /usr/local/etc/salt/master + cp /usr/local/etc/salt/minion.sample /usr/local/etc/salt/minion + +Note: You'll only need to copy the config for the service you're going to run. + +Once you've copied the config into place you'll need to make changes specific +to your setup. Below I'll outline suggested configuration changes to the +Master, after which I'll outline configuring the Minion. + +.. _master_configuration: + +Master Configuration +==================== + +This section outlines configuration of a Salt Master, which is used to control +other machines known as "minions" (see "Minion Configuration" for instructions +on configuring a minion). This will outline IP configuration, and a few key +configuration paths. + +**Interface** + +By default the Salt master listens on ports 4505 and 4506 on all interfaces +(0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the +"interface" directive as seen here:: + + - #interface: 0.0.0.0 + + interface: 10.0.0.1 + +**rc.conf** + +Last but not least you'll need to activate the Salt Master in your rc.conf +file. Using your favorite editor, open /etc/rc.conf or /etc/rc.conf.local and +add this line:: + + + salt_master_enable="YES" + +Once you've completed all of these steps you're ready to start your Salt +Master. The Salt port installs an rc script which should be used to manage your +Salt Master. You should be able to start your Salt Master now using the command +seen here:: + + service salt_master start + +If your Salt Master doesn't start successfully, go back through each step and +see if anything was missed. Salt doesn't take much configuration (part of its +beauty!), and errors are usually simple mistakes. + +.. _ minion_configuration: + +Minion Configuration +==================== + +Configuring a Salt Minion is surprisingly simple. Unless you have a real need +for customizing your minion configuration (which there are plenty of options if +you are so inclined!), there is one simple directive that needs to be updated. +That option is the location of the master. + +By default a Salt Minion will try to connect to the dns name "salt". If you +have the ability to update DNS records for your domain you might create an A or +CNAME record for "salt" that points to your Salt Master. If you are able to do +this you likely can do without any minion configuration at all. + +If you are not able to update DNS, you'll simply need to update one entry in +the configuration file. Using your favorite editor, open the minion +configuration file and update the "master" entry as seen here:: + + - #master: salt + + master: 10.0.0.1 + +Simply update the master directive to the IP or hostname of your Salt Master. +Save your changes and you're ready to start your Salt Minion. Advanced +configuration options are covered in another chapter. + +**rc.conf** + +Before you're able to start the Salt Minion you'll need to update your rc.conf +file. Using your favorite editor open /etc/rc.conf or /etc/rc.conf.local and +add this line:: + + + salt_minion_enable="YES" + +Once you've completed all of these steps you're ready to start your Salt +Minion. The Salt port installs an rc script which should be used to manage your +Salt Minion. You should be able to start your Salt Minion now using the command +seen here:: + + service salt_minion start + +If your Salt Minion doesn't start successfully, go back through each step and +see if anything was missed. Salt doesn't take much configuration (part of its +beauty!), and errors are usually simple mistakes. + +.. _tying_it_all_together: + +Tying It All Together +====================== + +If you've successfully completed each of the steps above you should have a +running Salt Master and a running Salt Minion. The Minion should be configured +to point to the Master. To verify that there is communication flowing between +the Minion and Master we'll run a few initial ``salt`` commands. These commands +will validate the Minions RSA encryption key, and then send a test command to +the Minion to ensure that commands and responses are flowing as expected. + +**Key Management** + +Salt uses AES encryption for all communication between the Master and the +Minion. This ensures that the commands you send to your Minions (your cloud) +can not be tampered with, and that communication between Master and Minion is +only done through trusted, accepted keys. + +Before you'll be able to do any remote execution or state management you'll +need to accept any pending keys on the Master. Run the ``salt-key`` command to +list the keys known to the Salt Master:: + + [root@master ~]# salt-key -L + Unaccepted Keys: + avon + bodie + bubbles + marlo + Accepted Keys: + +This example shows that the Salt Master is aware of four Minions, but none of +the keys have been accepted. To accept the keys and allow the Minions to be +controlled by the Master, again use the ``salt-key`` command:: + + [root@master ~]# salt-key -A + [root@master ~]# salt-key -L + Unaccepted Keys: + Accepted Keys: + avon + bodie + bubbles + marlo + +The ``salt-key`` command allows for signing keys individually or in bulk. The +example above, using ``-A`` bulk-accepts all pending keys. To accept keys +individually use the lowercase of the same option, ``-a keyname``. + +.. _sending_commands: + +Sending Commands +================ + +Everything should be set for you to begin remote management of your Minions. +Whether you have a few or a few-dozen, Salt can help you manage them easily! + +For final verification, send a test function from your Salt Master to your +minions. If all of your minions are properly communicating with your Master, +you should "True" responses from each of them. See the example below to send +the ``test.ping`` remote command:: + + [root@master ~]# salt '*' test.ping + {'avon': True} + +.. _where_do_i_go_from_here: + +Where Do I Go From Here +======================== + +Congratulations! You've successfully configured your first Salt Minions and are +able to send remote commands. I'm sure you're eager to learn more about what +Salt can do. Depending on the primary way you want to manage your machines you +may either want to visit the section regarding Salt States, or the section on +Modules. diff --git a/doc/topics/tutorials/archlinux.rst b/doc/topics/tutorials/archlinux.rst deleted file mode 100644 index 7af629fccd0e..000000000000 --- a/doc/topics/tutorials/archlinux.rst +++ /dev/null @@ -1,258 +0,0 @@ -.. _introduction: - -Introduction -============ - -Salt has primarily been developed on Arch Linux, meaning it is known to work -very well on that distribution. The lead developer, Thomas S. Hatch (thatch45) has -been a TU (Trusted User) for the Arch Linux distribution, and has written a -number of Arch-specific tools in the past. - -Salt, while not Arch-specific, is packaged for and works well on Arch Linux. - -.. _installation: - -Installation -============ - -Salt is currently available via the Arch User Repository (AUR). There are -currently stable and -git packages available. - -Stable Release --------------- - -To install Salt stable releases from the Arch Linux AUR, use the commands:: - - wget https://aur.archlinux.org/packages/sa/salt/salt.tar.gz - tar xf salt.tar.gz - cd salt/ - makepkg -is - -A few of Salt's dependencies are currently only found within the AUR, so you'll -need to download and run ``makepkg -is`` on these as well. As a reference, Salt -currently relies on the following packages only available via the AUR: - -* https://aur.archlinux.org/packages/py/python2-msgpack/python2-msgpack.tar.gz -* https://aur.archlinux.org/packages/py/python2-psutil/python2-psutil.tar.gz - -.. note:: yaourt - - If you chose to use a tool such as Yaourt_ the dependencies will be - gathered and built for you automatically. - - The command to install salt using the yaourt tool is: - - .. code-block:: none - - yaourt salt - -.. _Yaourt: https://aur.archlinux.org/packages.php?ID=5863 - -Tracking develop ----------------- - -To install the bleeding edge version of Salt (**may include bugs!**), you can use -the -git package. Installing the -git package can be done using the commands:: - - wget https://aur.archlinux.org/packages/sa/salt-git/salt-git.tar.gz - tar xf salt-git.tar.gz - cd salt-git/ - makepkg -is - -A few of Salt's dependencies are currently only found within the AUR, so you'll -need to download and run ``makepkg -is`` on these as well. As a reference, Salt -currently relies on the following packages only available via the AUR: - -* https://aur.archlinux.org/packages/py/python2-msgpack/python2-msgpack.tar.gz -* https://aur.archlinux.org/packages/py/python2-psutil/python2-psutil.tar.gz - -.. note:: yaourt - - If you chose to use a tool such as Yaourt_ the dependencies will be - gathered and built for you automatically. - - The command to install salt using the yaourt tool is: - - .. code-block:: none - - yaourt salt-git - -.. _Yaourt: https://aur.archlinux.org/packages.php?ID=5863 - -.. _configuration: - -Configuration -============= - -In the sections below I'll outline configuration options for both the Salt -Master and Salt Minions. - -The Salt package installs two template configuration files, /etc/salt/master.template and -/etc/salt/minion.template. You'll need to copy these .template files into place and -make a few edits. First, copy them into place as seen here:: - - cp /etc/salt/master.template /etc/salt/master - cp /etc/salt/minion.template /etc/salt/minion - -Note: You'll only need to copy the config for the service you're going to run. - -Once you've copied the config into place you'll need to make changes specific -to your setup. Below I'll outline suggested configuration changes to the -Master, after which I'll outline configuring the Minion. - -.. _master_configuration: - -Master Configuration -==================== - -This section outlines configuration of a Salt Master, which is used to control -other machines known as "minions" (see "Minion Configuration" for instructions -on configuring a minion). This will outline IP configuration, and a few key -configuration paths. - -**Interface** - -By default the Salt master listens on ports 4505 and 4506 on all interfaces -(0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the -"interface" directive as seen here:: - - - #interface: 0.0.0.0 - + interface: 10.0.0.1 - -**rc.conf** - -Last but not least you'll need to activate the Salt Master in your rc.conf -file. Using your favorite editor, open /etc/rc.conf and add the salt-master:: - - -DAEMONS=(syslog-ng network crond) - +DAEMONS=(syslog-ng network crond @salt-master) - -Once you've completed all of these steps you're ready to start your Salt -Master. You should be able to start your Salt Master now using the command -seen here:: - - rc.d start salt-master - -If your Salt Master doesn't start successfully, go back through each step and -see if anything was missed. Salt doesn't take much configuration (part of its -beauty!), and errors are usually simple mistakes. - -.. _ minion_configuration: - -Minion Configuration -==================== - -Configuring a Salt Minion is surprisingly simple. Unless you have a real need -for customizing your minion configuration (which there are plenty of options if -you are so inclined!), there is one simple directive that needs to be updated. -That option is the location of the master. - -By default a Salt Minion will try to connect to the dns name "salt". If you -have the ability to update DNS records for your domain you might create an A or -CNAME record for "salt" that points to your Salt Master. If you are able to do -this you likely can do without any minion configuration at all. - -If you are not able to update DNS, you'll simply need to update one entry in -the configuration file. Using your favorite editor, open the minion -configuration file and update the "master" entry as seen here:: - - - #master: salt - + master: 10.0.0.1 - -Simply update the master directive to the IP or hostname of your Salt Master. -Save your changes and you're ready to start your Salt Minion. Advanced -configuration options are covered in another chapter. - -**rc.conf** - -Before you're able to start the Salt Minion you'll need to update your rc.conf -file. Using your favorite editor open /etc/rc.conf or /etc/rc.conf.local and -add this line:: - - -DAEMONS=(syslog-ng network crond) - +DAEMONS=(syslog-ng network crond @salt-minion) - -Once you've completed all of these steps you're ready to start your Salt -Minion. You should be able to start your Salt Minion now using the command -seen here:: - - rc.d start salt-minion - -If your Salt Minion doesn't start successfully, go back through each step and -see if anything was missed. Salt doesn't take much configuration (part of its -beauty!), and errors are usually simple mistakes. - -.. _tying_it_all_together: - -Tying It All Together -====================== - -If you've successfully completed each of the steps above you should have a -running Salt Master and a running Salt Minion. The Minion should be configured -to point to the Master. To verify that there is communication flowing between -the Minion and Master we'll run a few initial ``salt`` commands. These commands -will validate the Minions RSA encryption key, and then send a test command to -the Minion to ensure that commands and responses are flowing as expected. - -**Key Management** - -Salt uses AES encryption for all communication between the Master and the -Minion. This ensures that the commands you send to your Minions (your cloud) -can not be tampered with, and that communication between Master and Minion is -only done through trusted, accepted keys. - -Before you'll be able to do any remote execution or configuration management you'll -need to accept any pending keys on the Master. Run the ``salt-key`` command to -list the keys known to the Salt Master:: - - [root@master ~]# salt-key -L - Unaccepted Keys: - avon - bodie - bubbles - marlo - Accepted Keys: - -This example shows that the Salt Master is aware of four Minions, but none of -the keys have been accepted. To accept the keys and allow the Minions to be -controlled by the Master, again use the ``salt-key`` command:: - - [root@master ~]# salt-key -A - [root@master ~]# salt-key -L - Unaccepted Keys: - Accepted Keys: - avon - bodie - bubbles - marlo - -The ``salt-key`` command allows for signing keys individually or in bulk. The -example above, using ``-A`` bulk-accepts all pending keys. To accept keys -individually use the lowercase of the same option, ``-a keyname``. - -.. _sending_commands: - -Sending Commands -================ - -Everything should be set for you to begin remote management of your Minions. -Whether you have a few or a few-dozen, Salt can help you manage them easily! - -For final verification, send a test function from your Salt Master to your -minions. If all of your minions are properly communicating with your Master, -you should "True" responses from each of them. See the example below to send -the ``test.ping`` remote command:: - - [root@master ~]# salt '*' test.ping - {'avon': True} - -.. _where_do_i_go_from_here: - -Where Do I Go From Here -======================== - -Congratulations! You've successfully configured your first Salt Minions and are -able to send remote commands. I'm sure you're eager to learn more about what -Salt can do. Depending on the primary way you want to manage your machines you -may either want to visit the section regarding Salt States, or the section on -Modules. diff --git a/doc/topics/tutorials/fedora.rst b/doc/topics/tutorials/fedora.rst deleted file mode 100644 index d38efc305a11..000000000000 --- a/doc/topics/tutorials/fedora.rst +++ /dev/null @@ -1,189 +0,0 @@ -.. _introduction: - -Introduction -============ - -Beginning with version 0.9.4, Salt has been available in the primary Fedora -repositories and is available for installation using yum. Fedora will have more -up to date versions of Salt than other members of the Red Hat family, which -makes it a great place to help improve Salt! - -.. _installation: - -Installation -============ - -Salt can be installed using ``yum`` and is available in the standard Fedora -repositories. - -Stable Release --------------- - -Salt is packaged separately for the minion and the master. You'll only need to -install the appropriate package for the role you need the machine to play. This -means you're going to want one master and a whole bunch of minions!:: - - yum install salt-master - yum install salt-minion - -.. _configuration: - -Configuration -============= - -In the sections below I'll outline configuration options for both the Salt -Master and Salt Minions. - -.. _master_configuration: - -Master Configuration -==================== - -This section outlines configuration of a Salt Master, which is used to control -other machines known as "minions" (see "Minion Configuration" for instructions -on configuring a minion). This will outline IP configuration, and a few key -configuration paths. - -**Interface** - -By default the Salt master listens on ports 4505 and 4506 on all interfaces -(0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the -"interface" directive as seen here:: - - - #interface: 0.0.0.0 - + interface: 10.0.0.1 - -**Enable the Master** - -You'll also likely want to activate the Salt Master in systemd, configuring the -Salt Master to start automatically at boot.:: - - systemctl enable salt-master.service - -Once you've completed all of these steps you're ready to start your Salt -Master. You should be able to start your Salt Master now using the command -seen here:: - - systemctl start salt-master.service - -If your Salt Master doesn't start successfully, go back through each step and -see if anything was missed. Salt doesn't take much configuration (part of its -beauty!), and errors are usually simple mistakes. - -.. _ minion_configuration: - -Minion Configuration -==================== - -Configuring a Salt Minion is surprisingly simple. Unless you have a real need -for customizing your minion configuration (which there are plenty of options if -you are so inclined!), there is one simple directive that needs to be updated. -That option is the location of the master. - -By default a Salt Minion will try to connect to the dns name "salt". If you -have the ability to update DNS records for your domain you might create an A or -CNAME record for "salt" that points to your Salt Master. If you are able to do -this you likely can do without any minion configuration at all. - -If you are not able to update DNS, you'll simply need to update one entry in -the configuration file. Using your favorite editor, open the minion -configuration file and update the "master" entry as seen here:: - - - #master: salt - + master: 10.0.0.1 - -Simply update the master directive to the IP or hostname of your Salt Master. -Save your changes and you're ready to start your Salt Minion. Advanced -configuration options are covered in another chapter. - -**Enable the Minion** - -You'll need to configure the minion to auto-start at boot. You can toggle -that option through systemd.:: - - systemctl enable salt-minion.service - -Once you've completed all of these steps you're ready to start your Salt -Minion. You should be able to start your Salt Minion now using the command -here:: - - systemctl start salt-minion.service - -If your Salt Minion doesn't start successfully, go back through each step and -see if anything was missed. Salt doesn't take much configuration (part of its -beauty!), and errors are usually simple mistakes. - -.. _tying_it_all_together: - -Tying It All Together -====================== - -If you've successfully completed each of the steps above you should have a -running Salt Master and a running Salt Minion. The Minion should be configured -to point to the Master. To verify that there is communication flowing between -the Minion and Master we'll run a few initial ``salt`` commands. These commands -will validate the Minions RSA encryption key, and then send a test command to -the Minion to ensure that commands and responses are flowing as expected. - -**Key Management** - -Salt uses AES encryption for all communication between the Master and the -Minion. This ensures that the commands you send to your Minions (your cloud) -can not be tampered with, and that communication between Master and Minion is -only done through trusted, accepted keys. - -Before you'll be able to do any remote execution or configuration management you'll -need to accept any pending keys on the Master. Run the ``salt-key`` command to -list the keys known to the Salt Master:: - - [root@master ~]# salt-key -L - Unaccepted Keys: - avon - bodie - bubbles - marlo - Accepted Keys: - -This example shows that the Salt Master is aware of four Minions, but none of -the keys have been accepted. To accept the keys and allow the Minions to be -controlled by the Master, again use the ``salt-key`` command:: - - [root@master ~]# salt-key -A - [root@master ~]# salt-key -L - Unaccepted Keys: - Accepted Keys: - avon - bodie - bubbles - marlo - -The ``salt-key`` command allows for signing keys individually or in bulk. The -example above, using ``-A`` bulk-accepts all pending keys. To accept keys -individually use the lowercase of the same option, ``-a keyname``. - -.. _sending_commands: - -Sending Commands -================ - -Everything should be set for you to begin remote management of your Minions. -Whether you have a few or a few-dozen, Salt can help you manage them easily! - -For final verification, send a test function from your Salt Master to your -minions. If all of your minions are properly communicating with your Master, -you should "True" responses from each of them. See the example below to send -the ``test.ping`` remote command:: - - [root@master ~]# salt '*' test.ping - {'avon': True} - -.. _where_do_i_go_from_here: - -Where Do I Go From Here -======================== - -Congratulations! You've successfully configured your first Salt Minions and are -able to send remote commands. I'm sure you're eager to learn more about what -Salt can do. Depending on the primary way you want to manage your machines you -may either want to visit the section regarding Salt States, or the section on -Modules. diff --git a/doc/topics/tutorials/freebsd.rst b/doc/topics/tutorials/freebsd.rst deleted file mode 100644 index 6eb02ecf46cf..000000000000 --- a/doc/topics/tutorials/freebsd.rst +++ /dev/null @@ -1,211 +0,0 @@ -.. _introduction: - -Introduction -============ - -Salt was added to the FreeBSD ports tree Dec 26th, 2011 by Christer Edwards -<christer.edwards@gmail.com>. It has been tested on FreeBSD 8.2 and 9.0 -releases. - -Salt is dependent on the following additional ports. These will be installed as -dependencies of the ``sysutils/salt`` port:: - - /devel/py-yaml - /devel/py-pyzmq - /devel/py-Jinja2 - /devel/py-msgpack - /security/py-pycrypto - /security/py-m2crypto - -.. _installation: - -Installation -============ - -To install Salt from the FreeBSD ports tree, use the command:: - - cd /usr/ports/sysutils/salt && make install clean - -Once the port is installed you'll need to make a few configuration changes. -These include defining the IP to bind to (optional), and some configuration -path changes to make salt fit more natively into the FreeBSD filesystem tree. - -.. _configuration: - -Configuration -============= - -In the sections below I'll outline configuration options for both the Salt -Master and Salt Minions. - -The Salt port installs two sample configuration files, salt/master.sample and -salt/minion.sample (these should be installed in /usr/local/etc/, unless you use a -different %%PREFIX%%). You'll need to copy these .sample files into place and -make a few edits. First, copy them into place as seen here:: - - cp /usr/local/etc/salt/master.sample /usr/local/etc/salt/master - cp /usr/local/etc/salt/minion.sample /usr/local/etc/salt/minion - -Note: You'll only need to copy the config for the service you're going to run. - -Once you've copied the config into place you'll need to make changes specific -to your setup. Below I'll outline suggested configuration changes to the -Master, after which I'll outline configuring the Minion. - -.. _master_configuration: - -Master Configuration -==================== - -This section outlines configuration of a Salt Master, which is used to control -other machines known as "minions" (see "Minion Configuration" for instructions -on configuring a minion). This will outline IP configuration, and a few key -configuration paths. - -**Interface** - -By default the Salt master listens on ports 4505 and 4506 on all interfaces -(0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the -"interface" directive as seen here:: - - - #interface: 0.0.0.0 - + interface: 10.0.0.1 - -**rc.conf** - -Last but not least you'll need to activate the Salt Master in your rc.conf -file. Using your favorite editor, open /etc/rc.conf or /etc/rc.conf.local and -add this line:: - - + salt_master_enable="YES" - -Once you've completed all of these steps you're ready to start your Salt -Master. The Salt port installs an rc script which should be used to manage your -Salt Master. You should be able to start your Salt Master now using the command -seen here:: - - service salt_master start - -If your Salt Master doesn't start successfully, go back through each step and -see if anything was missed. Salt doesn't take much configuration (part of its -beauty!), and errors are usually simple mistakes. - -.. _ minion_configuration: - -Minion Configuration -==================== - -Configuring a Salt Minion is surprisingly simple. Unless you have a real need -for customizing your minion configuration (which there are plenty of options if -you are so inclined!), there is one simple directive that needs to be updated. -That option is the location of the master. - -By default a Salt Minion will try to connect to the dns name "salt". If you -have the ability to update DNS records for your domain you might create an A or -CNAME record for "salt" that points to your Salt Master. If you are able to do -this you likely can do without any minion configuration at all. - -If you are not able to update DNS, you'll simply need to update one entry in -the configuration file. Using your favorite editor, open the minion -configuration file and update the "master" entry as seen here:: - - - #master: salt - + master: 10.0.0.1 - -Simply update the master directive to the IP or hostname of your Salt Master. -Save your changes and you're ready to start your Salt Minion. Advanced -configuration options are covered in another chapter. - -**rc.conf** - -Before you're able to start the Salt Minion you'll need to update your rc.conf -file. Using your favorite editor open /etc/rc.conf or /etc/rc.conf.local and -add this line:: - - + salt_minion_enable="YES" - -Once you've completed all of these steps you're ready to start your Salt -Minion. The Salt port installs an rc script which should be used to manage your -Salt Minion. You should be able to start your Salt Minion now using the command -seen here:: - - service salt_minion start - -If your Salt Minion doesn't start successfully, go back through each step and -see if anything was missed. Salt doesn't take much configuration (part of its -beauty!), and errors are usually simple mistakes. - -.. _tying_it_all_together: - -Tying It All Together -====================== - -If you've successfully completed each of the steps above you should have a -running Salt Master and a running Salt Minion. The Minion should be configured -to point to the Master. To verify that there is communication flowing between -the Minion and Master we'll run a few initial ``salt`` commands. These commands -will validate the Minions RSA encryption key, and then send a test command to -the Minion to ensure that commands and responses are flowing as expected. - -**Key Management** - -Salt uses AES encryption for all communication between the Master and the -Minion. This ensures that the commands you send to your Minions (your cloud) -can not be tampered with, and that communication between Master and Minion is -only done through trusted, accepted keys. - -Before you'll be able to do any remote execution or state management you'll -need to accept any pending keys on the Master. Run the ``salt-key`` command to -list the keys known to the Salt Master:: - - [root@master ~]# salt-key -L - Unaccepted Keys: - avon - bodie - bubbles - marlo - Accepted Keys: - -This example shows that the Salt Master is aware of four Minions, but none of -the keys have been accepted. To accept the keys and allow the Minions to be -controlled by the Master, again use the ``salt-key`` command:: - - [root@master ~]# salt-key -A - [root@master ~]# salt-key -L - Unaccepted Keys: - Accepted Keys: - avon - bodie - bubbles - marlo - -The ``salt-key`` command allows for signing keys individually or in bulk. The -example above, using ``-A`` bulk-accepts all pending keys. To accept keys -individually use the lowercase of the same option, ``-a keyname``. - -.. _sending_commands: - -Sending Commands -================ - -Everything should be set for you to begin remote management of your Minions. -Whether you have a few or a few-dozen, Salt can help you manage them easily! - -For final verification, send a test function from your Salt Master to your -minions. If all of your minions are properly communicating with your Master, -you should "True" responses from each of them. See the example below to send -the ``test.ping`` remote command:: - - [root@master ~]# salt '*' test.ping - {'avon': True} - -.. _where_do_i_go_from_here: - -Where Do I Go From Here -======================== - -Congratulations! You've successfully configured your first Salt Minions and are -able to send remote commands. I'm sure you're eager to learn more about what -Salt can do. Depending on the primary way you want to manage your machines you -may either want to visit the section regarding Salt States, or the section on -Modules. From be81d5dde585dc64624ad62021181aa497aad6ec Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Sat, 25 Feb 2012 14:43:45 -0700 Subject: [PATCH 091/598] Removed duplicate global labels from install tutorials --- doc/topics/installation/arch.rst | 16 ---------------- doc/topics/installation/fedora.rst | 16 ---------------- doc/topics/installation/freebsd.rst | 16 ---------------- 3 files changed, 48 deletions(-) diff --git a/doc/topics/installation/arch.rst b/doc/topics/installation/arch.rst index 7af629fccd0e..95418f212173 100644 --- a/doc/topics/installation/arch.rst +++ b/doc/topics/installation/arch.rst @@ -1,5 +1,3 @@ -.. _introduction: - Introduction ============ @@ -10,8 +8,6 @@ number of Arch-specific tools in the past. Salt, while not Arch-specific, is packaged for and works well on Arch Linux. -.. _installation: - Installation ============ @@ -79,8 +75,6 @@ currently relies on the following packages only available via the AUR: .. _Yaourt: https://aur.archlinux.org/packages.php?ID=5863 -.. _configuration: - Configuration ============= @@ -100,8 +94,6 @@ Once you've copied the config into place you'll need to make changes specific to your setup. Below I'll outline suggested configuration changes to the Master, after which I'll outline configuring the Minion. -.. _master_configuration: - Master Configuration ==================== @@ -137,8 +129,6 @@ If your Salt Master doesn't start successfully, go back through each step and see if anything was missed. Salt doesn't take much configuration (part of its beauty!), and errors are usually simple mistakes. -.. _ minion_configuration: - Minion Configuration ==================== @@ -182,8 +172,6 @@ If your Salt Minion doesn't start successfully, go back through each step and see if anything was missed. Salt doesn't take much configuration (part of its beauty!), and errors are usually simple mistakes. -.. _tying_it_all_together: - Tying It All Together ====================== @@ -230,8 +218,6 @@ The ``salt-key`` command allows for signing keys individually or in bulk. The example above, using ``-A`` bulk-accepts all pending keys. To accept keys individually use the lowercase of the same option, ``-a keyname``. -.. _sending_commands: - Sending Commands ================ @@ -246,8 +232,6 @@ the ``test.ping`` remote command:: [root@master ~]# salt '*' test.ping {'avon': True} -.. _where_do_i_go_from_here: - Where Do I Go From Here ======================== diff --git a/doc/topics/installation/fedora.rst b/doc/topics/installation/fedora.rst index 186514e57637..3e78da4875d4 100644 --- a/doc/topics/installation/fedora.rst +++ b/doc/topics/installation/fedora.rst @@ -1,5 +1,3 @@ -.. _introduction: - Introduction ============ @@ -17,8 +15,6 @@ makes it a great place to help improve Salt! wget -O /etc/yum.repos.d/epel-salt.repo \\ http://repos.fedorapeople.org/repos/herlo/salt/epel-salt.repo -.. _installation: - Installation ============ @@ -35,16 +31,12 @@ means you're going to want one master and a whole bunch of minions!:: yum install salt-master yum install salt-minion -.. _configuration: - Configuration ============= In the sections below I'll outline configuration options for both the Salt Master and Salt Minions. -.. _master_configuration: - Master Configuration ==================== @@ -79,8 +71,6 @@ If your Salt Master doesn't start successfully, go back through each step and see if anything was missed. Salt doesn't take much configuration (part of its beauty!), and errors are usually simple mistakes. -.. _ minion_configuration: - Minion Configuration ==================== @@ -122,8 +112,6 @@ If your Salt Minion doesn't start successfully, go back through each step and see if anything was missed. Salt doesn't take much configuration (part of its beauty!), and errors are usually simple mistakes. -.. _tying_it_all_together: - Tying It All Together ====================== @@ -170,8 +158,6 @@ The ``salt-key`` command allows for signing keys individually or in bulk. The example above, using ``-A`` bulk-accepts all pending keys. To accept keys individually use the lowercase of the same option, ``-a keyname``. -.. _sending_commands: - Sending Commands ================ @@ -186,8 +172,6 @@ the ``test.ping`` remote command:: [root@master ~]# salt '*' test.ping {'avon': True} -.. _where_do_i_go_from_here: - Where Do I Go From Here ======================== diff --git a/doc/topics/installation/freebsd.rst b/doc/topics/installation/freebsd.rst index 6eb02ecf46cf..cfb6f0c6019a 100644 --- a/doc/topics/installation/freebsd.rst +++ b/doc/topics/installation/freebsd.rst @@ -1,5 +1,3 @@ -.. _introduction: - Introduction ============ @@ -17,8 +15,6 @@ dependencies of the ``sysutils/salt`` port:: /security/py-pycrypto /security/py-m2crypto -.. _installation: - Installation ============ @@ -30,8 +26,6 @@ Once the port is installed you'll need to make a few configuration changes. These include defining the IP to bind to (optional), and some configuration path changes to make salt fit more natively into the FreeBSD filesystem tree. -.. _configuration: - Configuration ============= @@ -52,8 +46,6 @@ Once you've copied the config into place you'll need to make changes specific to your setup. Below I'll outline suggested configuration changes to the Master, after which I'll outline configuring the Minion. -.. _master_configuration: - Master Configuration ==================== @@ -90,8 +82,6 @@ If your Salt Master doesn't start successfully, go back through each step and see if anything was missed. Salt doesn't take much configuration (part of its beauty!), and errors are usually simple mistakes. -.. _ minion_configuration: - Minion Configuration ==================== @@ -135,8 +125,6 @@ If your Salt Minion doesn't start successfully, go back through each step and see if anything was missed. Salt doesn't take much configuration (part of its beauty!), and errors are usually simple mistakes. -.. _tying_it_all_together: - Tying It All Together ====================== @@ -183,8 +171,6 @@ The ``salt-key`` command allows for signing keys individually or in bulk. The example above, using ``-A`` bulk-accepts all pending keys. To accept keys individually use the lowercase of the same option, ``-a keyname``. -.. _sending_commands: - Sending Commands ================ @@ -199,8 +185,6 @@ the ``test.ping`` remote command:: [root@master ~]# salt '*' test.ping {'avon': True} -.. _where_do_i_go_from_here: - Where Do I Go From Here ======================== From bca4c2ff12b83f6f275a8c82346bc2d8395e32f1 Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Sat, 25 Feb 2012 14:51:46 -0700 Subject: [PATCH 092/598] Added document-level headings to platform-specific headings --- doc/topics/installation/arch.rst | 5 +++-- doc/topics/installation/fedora.rst | 5 +++-- doc/topics/installation/freebsd.rst | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/doc/topics/installation/arch.rst b/doc/topics/installation/arch.rst index 95418f212173..24c8a884bbec 100644 --- a/doc/topics/installation/arch.rst +++ b/doc/topics/installation/arch.rst @@ -1,5 +1,6 @@ -Introduction -============ +========== +Arch Linux +========== Salt has primarily been developed on Arch Linux, meaning it is known to work very well on that distribution. The lead developer, Thomas S. Hatch (thatch45) has diff --git a/doc/topics/installation/fedora.rst b/doc/topics/installation/fedora.rst index 3e78da4875d4..cdfe980f4056 100644 --- a/doc/topics/installation/fedora.rst +++ b/doc/topics/installation/fedora.rst @@ -1,5 +1,6 @@ -Introduction -============ +================================== +Fedora & CentOS / Enterprise Linux +================================== Beginning with version 0.9.4, Salt has been available in the primary Fedora repositories and is available for installation using yum. Fedora will have more diff --git a/doc/topics/installation/freebsd.rst b/doc/topics/installation/freebsd.rst index cfb6f0c6019a..043d90078c24 100644 --- a/doc/topics/installation/freebsd.rst +++ b/doc/topics/installation/freebsd.rst @@ -1,5 +1,6 @@ -Introduction -============ +======= +FreeBSD +======= Salt was added to the FreeBSD ports tree Dec 26th, 2011 by Christer Edwards <christer.edwards@gmail.com>. It has been tested on FreeBSD 8.2 and 9.0 From f01a76546e9141c10e161380e5f85e55e3be9491 Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Sat, 25 Feb 2012 14:48:04 -0700 Subject: [PATCH 093/598] Removed old grains doc from toctree Should have been in 084f8a7. --- doc/contents.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/contents.rst b/doc/contents.rst index 666331a0d579..8ef6c75b7b40 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -17,7 +17,6 @@ Full Table of Contents ref/index ref/modules/* ref/modules/all/index - ref/grains ref/returners/* ref/returners/all/index ref/states/* From 1211cfff34fc53b6bba97a81b73ffa2de48db373 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 15:00:40 -0700 Subject: [PATCH 094/598] Add openvz ps option --- salt/grains/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/grains/core.py b/salt/grains/core.py index 491995433064..3e6c19ab4f28 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -257,6 +257,8 @@ def _ps(osdata): bsd_choices = ('FreeBSD', 'NetBSD', 'OpenBSD', 'Darwin') if osdata['os'] in bsd_choices: grains['ps'] = 'ps auxwww' + elif osdata.get('virtual', '') == 'openvzhn': + grains['ps'] = 'vzps -E 0 -efH|cut -b 6-' else: grains['ps'] = 'ps -efH' return grains From df4c8ed7608c6f4478cdcc4865df6a0f8701f875 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 15:15:05 -0700 Subject: [PATCH 095/598] Add run_num to 0 sls file return --- salt/state.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 6de2d2cbebeb..7b36dc0a5ce8 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1086,7 +1086,8 @@ def call_highstate(self): 'result': False, 'comment': 'No states found for this minion', 'name': 'No States', - 'changes': {} + 'changes': {}, + '__run_num__': 0, } } return self.state.call_high(high) From 69958807ad6814eda16fd29369de4e1db7da22a8 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 17:00:48 -0700 Subject: [PATCH 096/598] The shadow module != file --- salt/modules/shadow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/shadow.py b/salt/modules/shadow.py index 2d7338a045cc..8bc478ab7718 100644 --- a/salt/modules/shadow.py +++ b/salt/modules/shadow.py @@ -13,7 +13,7 @@ def __virtual__(): # Disable on Windows, a specific file module exists: if __grains__['os'] == 'Windows': return False - return 'file' + return 'shadow' def info(name): From 17f2ac717dc6d21d1913e025f122931d3194557d Mon Sep 17 00:00:00 2001 From: Eivind Uggedal <eivind@uggedal.com> Date: Sun, 26 Feb 2012 01:36:00 +0100 Subject: [PATCH 097/598] Fix bug where only the last extend across a highstate was realized. --- salt/state.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/state.py b/salt/state.py index 7b36dc0a5ce8..ace0d62ddbbd 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1054,6 +1054,12 @@ def render_highstate(self, matches): mods = set() for sls in states: state, mods, err = self.render_state(sls, env, mods) + if '__extend__' in state: + ext = state.pop('__extend__') + if '__extend__' in highstate: + highstate['__extend__'].extend(ext) + else: + highstate['__extend__'] = ext for id_ in state: if id_ in highstate: if highstate[id_] != state[id_]: From 706b980e6696a4db399fc15d5be9d13b5aa10363 Mon Sep 17 00:00:00 2001 From: Eivind Uggedal <eivind@uggedal.com> Date: Sun, 26 Feb 2012 01:41:48 +0100 Subject: [PATCH 098/598] More concise handling of extend members across states. --- salt/state.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/salt/state.py b/salt/state.py index ace0d62ddbbd..3bc296f8a50d 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1054,12 +1054,9 @@ def render_highstate(self, matches): mods = set() for sls in states: state, mods, err = self.render_state(sls, env, mods) - if '__extend__' in state: - ext = state.pop('__extend__') - if '__extend__' in highstate: - highstate['__extend__'].extend(ext) - else: - highstate['__extend__'] = ext + # The extend members can not be treated as globally unique: + if '__extend__' in state and '__extend__' in highstate: + highstate['__extend__'].extend(state.pop('__extend__')) for id_ in state: if id_ in highstate: if highstate[id_] != state[id_]: From ef484d39aa485dab39ea445cbb3fa220aab18508 Mon Sep 17 00:00:00 2001 From: Erik Johnson <palehose@gmail.com> Date: Sat, 25 Feb 2012 19:29:43 -0600 Subject: [PATCH 099/598] add recursive ownership management to file.directory --- salt/states/file.py | 103 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index 7be41063da69..4c3cb1788108 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -41,8 +41,24 @@ - mode: 755 - makedirs: True +If you need to enforce user and/or group ownership recursively on the +directory's contents, you can do so by adding a ``recurse`` directive: + +.. code-block:: yaml + + /srv/stuff/substuf: + file: + - directory + - user: fred + - group: users + - mode: 755 + - makedirs: True + - recurse: + - user + - group + Symlinks can be easily created, the symlink function is very simple and only -takes a few arguments +takes a few arguments: .. code-block:: yaml @@ -75,6 +91,7 @@ import tempfile import traceback import urlparse +import copy logger = logging.getLogger(__name__) @@ -759,6 +776,7 @@ def managed(name, def directory(name, user=None, group=None, + recurse=[], mode=None, makedirs=False, clean=False, @@ -777,6 +795,9 @@ def directory(name, The group ownership set for the directory, this defaults to the group salt is running as on the minion + recurse + Enforce user/group ownership of directory recursively + mode The permissions to set on this directory, aka 755 @@ -864,6 +885,86 @@ def directory(name, elif 'cgroup' in perms: ret['changes']['group'] = group + if recurse: + if not set(['user','group']) >= set(recurse): + ret['result'] = False + ret['comment'] = 'Types for "recurse" limited to "user" and ' \ + '"group"' + else: + targets = copy.copy(recurse) + if 'user' in targets: + if user: + uid = __salt__['file.user_to_uid'](user) + # file.user_to_uid returns '' if user does not exist. Above + # check for user is not fatal, so we need to be sure user + # exists. + if type(uid).__name__ == 'str': + ret['result'] = False + ret['comment'] = 'Failed to enforce ownership for ' \ + 'user {0} (user does not ' \ + 'exist)'.format(user) + # Remove 'user' from list of recurse targets + targets = filter(lambda x: x != 'user', targets) + else: + ret['result'] = False + ret['comment'] = 'user not specified, but configured as ' \ + 'a target for recursive ownership management' + # Remove 'user' from list of recurse targets + targets = filter(lambda x: x != 'user', targets) + if 'group' in targets: + if group: + gid = __salt__['file.group_to_gid'](group) + # As above with user, we need to make sure group exists. + if type(gid).__name__ == 'str': + ret['result'] = False + ret['comment'] = 'Failed to enforce group ownership ' \ + 'for group {0}'.format(group,user) + # Remove 'group' from list of recurse targets + targets = filter(lambda x: x != 'group', targets) + else: + ret['result'] = False + ret['comment'] = 'group not specified, but configured ' \ + 'as a target for recursive ownership management' + # Remove 'group' from list of recurse targets + targets = filter(lambda x: x != 'group', targets) + + needs_fixed = {} + if targets: + file_tree = __salt__['file.find'](name) + for path in file_tree: + fstat = os.stat(path) + if 'user' in targets and fstat.st_uid != uid: + needs_fixed['user'] = True + if needs_fixed.get('group'): break + if 'group' in targets and fstat.st_gid != gid: + needs_fixed['group'] = True + if needs_fixed.get('user'): break + + if needs_fixed.get('user'): + # Make sure the 'recurse' subdict exists + ret['changes'].setdefault('recurse',{}) + if 'user' in targets: + if __salt__['cmd.retcode']('chown -R {0} "{1}"'.format( + user,name)) != 0: + ret['result'] = False + ret['comment'] = 'Failed to enforce ownership on ' \ + '{0} for user {1}'.format(name,group) + else: + ret['changes']['recurse']['user'] = \ + __salt__['file.uid_to_user'](uid) + if needs_fixed.get('group'): + ret['changes'].setdefault('recurse',{}) + if 'group' in targets: + if __salt__['cmd.retcode']('chown -R :{0} "{1}"'.format( + group,name)) != 0: + ret['result'] = False + ret['comment'] = 'Failed to enforce group ownership ' \ + 'on {0} for group ' \ + '{1}'.format(name,group) + else: + ret['changes']['recurse']['group'] = \ + __salt__['file.gid_to_group'](gid) + if clean: keep = _gen_keep_files(name, require) removed = _clean_dir(name, list(keep)) From a7251ba856f01fa10bfbbd8a95f77e8bdde99b8e Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Sat, 25 Feb 2012 20:51:52 -0700 Subject: [PATCH 100/598] Basic Windows Registry management Squashed commit of the following: commit 6cf776ebbb7d742934a0ab2bd823fd360b8868ce Author: David Boucha <boucha@gmail.com> Date: Sat Feb 25 20:30:36 2012 -0700 Basic Windows Registry management completed commit f18a2e99396bcc0154fc0a20be8800f166f5acb5 Author: David Boucha <boucha@gmail.com> Date: Sat Feb 25 17:34:54 2012 -0700 Add reg.read_key commit e8cecf96c31f9c2a545614a5a901c26125197887 Author: David Boucha <boucha@gmail.com> Date: Sat Feb 25 15:25:09 2012 -0700 Add reg.py for managing Registry settings in Windows --- salt/modules/reg.py | 101 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 salt/modules/reg.py diff --git a/salt/modules/reg.py b/salt/modules/reg.py new file mode 100644 index 000000000000..ba3e5b3a18f5 --- /dev/null +++ b/salt/modules/reg.py @@ -0,0 +1,101 @@ +''' +Manage the registry on Windows +''' + +import _winreg + +def __virtual__(): + ''' + Only works on Windows systems + ''' + if __grains__['os'] == 'Windows': + return 'reg' + return False + + +hkeys = {'HKEY_CURRENT_USER': _winreg.HKEY_CURRENT_USER, + 'HKEY_LOCAL_MACHINE': _winreg.HKEY_LOCAL_MACHINE, + 'HKEY_USERS': _winreg.HKEY_USERS, + } + + +def read_key(hkey, path, key): + ''' + Read registry key value + + CLI Example:: + + salt '*' reg.read_key HKEY_LOCAL_MACHINE 'SOFTWARE\\Salt' 'version' + ''' + hkey2 = hkeys[hkey] + fullpath = '\\\\'.join([path, key]) + try: + handle = _winreg.OpenKey(hkey2, fullpath, 0, _winreg.KEY_READ) + return _winreg.QueryValueEx(handle, key)[0] + except: + return False + + +def set_key(hkey, path, key, value): + ''' + Set a registry key + + CLI Example:: + + salt '*' reg.set_key HKEY_CURRENT_USER 'SOFTWARE\\Salt' 'version' '0.97' + ''' + hkey2 = hkeys[hkey] + fullpath = '\\\\'.join([path, key]) + try: + handle = _winreg.OpenKey(hkey2, fullpath, 0, _winreg.KEY_ALL_ACCESS) + _winreg.SetValueEx(handle, key, 0, _winreg.REG_SZ, value) + _winreg.CloseKey(handle) + return True + except: + handle = _winreg.CreateKey(hkey2, fullpath) + _winreg.SetValueEx(handle, key, 0, _winreg.REG_SZ, value) + _winreg.CloseKey(handle) + return True + + +def create_key(hkey, path, key, value=None): + ''' + Create a registry key + + CLI Example:: + + salt '*' reg.create_key HKEY_CURRENT_USER 'SOFTWARE\\Salt' 'version' '0.97' + ''' + hkey2 = hkeys[hkey] + fullpath = '\\\\'.join([path, key]) + try: + handle = _winreg.OpenKey(hkey2, fullpath, 0, _winreg.KEY_ALL_ACCESS) + _winreg.CloseKey(handle) + return True + except: + handle = _winreg.CreateKey(hkey2, fullpath) + if value: + _winreg.SetValueEx(handle, key, 0, _winreg.REG_SZ, value) + _winreg.CloseKey(handle) + return True + + +def delete_key(hkey, path, key): + ''' + Delete a registry key + + Note: This cannot delete a key with subkeys + + CLI Example:: + + salt '*' reg.delete_key HKEY_CURRENT_USER 'SOFTWARE\\Salt' 'version' + ''' + hkey2 = hkeys[hkey] + try: + handle = _winreg.OpenKey(hkey2, path, 0, _winreg.KEY_ALL_ACCESS) + _winreg.DeleteKeyEx(handle, key) + _winreg.CloseKey(handle) + return True + except: + _winreg.CloseKey(handle) + return True From 632336ffbbbc76e8dfb5c5d35a9d5925c6d7a476 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 23:19:09 -0700 Subject: [PATCH 101/598] Add dynamic module loading doc --- doc/ref/file_server/dynamic-modules.rst | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/doc/ref/file_server/dynamic-modules.rst b/doc/ref/file_server/dynamic-modules.rst index fffff4acc44c..c204d59d8790 100644 --- a/doc/ref/file_server/dynamic-modules.rst +++ b/doc/ref/file_server/dynamic-modules.rst @@ -1,6 +1,37 @@ +=========================== Dynamic Module Distribution =========================== .. versionadded:: 0.9.5 +Salt python modules can be distributed automatically via the salt file server. +Under the root of any environment defined via the file_roots option on the +master server directories corresponding to the type of module can be used. + +The directories are prepended with an underscore: + + 1. _modules + 2. _grains + 3. _renderers + 4. _returners + 5. _states + +The contents of these directories need to be synced over to the minions after +python modules have been created in them. There are a number of ways to sync +the modules. + +Sync Via States +=============== + +The minion configuration contains an option ``autoload_dynamic_modules`` +which defaults to True. This option makes the state system refresh all +dynamic modules when states are run. To disable this behavior set +``autoload_dynamic_modules`` to False in the minion config. + +Sync Via the saltutil Module +============================ +The saltutil module has a number of functions that can be used to sync all +or specific dynamic modules. The saltutil module function ``saltutil.sync_all`` +will sync all module types over to a minion. For more information see: +:mod:`salt.modules.saltutil` From e1bcb75a714a597eafc0f7b0f23d7e7fe3be11a3 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 23:37:48 -0700 Subject: [PATCH 102/598] Add initial upstart support for Ubuntu --- salt/modules/service.py | 1 + salt/modules/upstart.py | 76 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 salt/modules/upstart.py diff --git a/salt/modules/service.py b/salt/modules/service.py index 3fd8f8807d29..5aee6db4ca21 100644 --- a/salt/modules/service.py +++ b/salt/modules/service.py @@ -26,6 +26,7 @@ def __virtual__(): 'Scientific', 'Fedora', 'Gentoo', + 'Ubuntu', 'FreeBSD', 'Windows', ] diff --git a/salt/modules/upstart.py b/salt/modules/upstart.py new file mode 100644 index 000000000000..6135050e7037 --- /dev/null +++ b/salt/modules/upstart.py @@ -0,0 +1,76 @@ +''' +Module for the management of upstart systems. The Upstart system only supports +service starting, stopping and restarting. + +DO NOT use this module on red hat systems, as red hat systems should use the +rh_service module, since red hat systems support chkconfig +''' + +import os + + +def __virtual__(): + ''' + Only work on Ubuntu + ''' + # Disable on these platforms, specific service modules exist: + if __grains__['os'] == 'Ubuntu': + return 'service' + return False + + +def _runlevel(): + ''' + Return the current runlevel + ''' + out = __salt__['cmd.run']('runlevel').strip() + return out.split()[1] + + +def start(name): + ''' + Start the specified service + + CLI Example:: + + salt '*' service.start <service name> + ''' + cmd = 'service {0} start'.format(name) + return not __salt__['cmd.retcode'](cmd) + + +def stop(name): + ''' + Stop the specified service + + CLI Example:: + + salt '*' service.stop <service name> + ''' + cmd = 'service {0} stop'.format(name) + return not __salt__['cmd.retcode'](cmd) + + +def restart(name): + ''' + Restart the named service + + CLI Example:: + + salt '*' service.restart <service name> + ''' + cmd = 'service {0} restart'.format(name) + return not __salt__['cmd.retcode'](cmd) + + +def status(name, sig=None): + ''' + Return the status for a service, returns a bool whether the service is + running. + + CLI Example:: + + salt '*' service.status <service name> + ''' + cmd = 'service {0} status'.format(name) + return not __salt__['cmd.retcode'](cmd) From d4d1e6a2fa946f1977bdb0931ea595d9b74f4038 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 23:39:17 -0700 Subject: [PATCH 103/598] fic docstring in rh_service module --- salt/modules/rh_service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/rh_service.py b/salt/modules/rh_service.py index 541efb110eb6..b5f8eaa4de67 100644 --- a/salt/modules/rh_service.py +++ b/salt/modules/rh_service.py @@ -1,6 +1,7 @@ ''' -Top level package command wrapper, used to translate the os detected by the -grains to the correct service manager +Service support for classic Red Hat type systems. This interface uses the +service command (so it is compatible with upstart systems) and the chkconfig +command. ''' import os From a010ffa9f30f4d13e561e107ee180e312ac86442 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 23:41:41 -0700 Subject: [PATCH 104/598] Add deprecation warning to manpage for salt -Q --- doc/ref/cli/salt.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/ref/cli/salt.rst b/doc/ref/cli/salt.rst index cf1af8db46c5..0f872fe51218 100644 --- a/doc/ref/cli/salt.rst +++ b/doc/ref/cli/salt.rst @@ -80,6 +80,10 @@ Options .. option:: -Q, --query + The -Q option is being deprecated and will be removed in a future release, + Use the salt jobs interface instead, for documentation on the salt jobs + interface execute the command "salt-run -d jobs" + Execute a salt command query, this can be used to find the results of a previous function call: -Q test.echo') From 0ffdcfe93181aba69fb429fc9547fec8e4ca2d93 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 25 Feb 2012 23:46:30 -0700 Subject: [PATCH 105/598] Add deprecation tot he salt -Q option --- salt/cli/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index eb1a5223070f..31ccabc8e77b 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -113,7 +113,9 @@ def __parse(self): dest='query', default=False, action='store_true', - help=('Execute a salt command query, this can be used to find ' + help=('This option is deprecated and will be removed in a ' + 'future release, please use salt-run jobs instead\n' + 'Execute a salt command query, this can be used to find ' 'the results os a previous function call: -Q test.echo')) parser.add_option('-c', '--config', From 43de62c8553ab2ff8d217673c706fd64bac042f8 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 26 Feb 2012 10:22:52 -0700 Subject: [PATCH 106/598] fix issue where grains don't always load static config content --- salt/loader.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/salt/loader.py b/salt/loader.py index 2234b8c79ecc..4bfd3ef18741 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -124,6 +124,22 @@ def grains(opts): module_dirs = [ os.path.join(salt_base_path, 'grains'), ] + extra_dirs + if not 'grains' in opts: + pre_opts = {} + salt.config.load_config( + pre_opts, + opts['conf_file'], + 'SALT_MINION_CONFIG' + ) + if 'include' in pre_opts: + pre_opts = salt.config.include_config( + pre_opts, + opts['conf_file'] + ) + if 'grains' in pre_opts: + opts['grains'] = pre_opts['grains'] + else: + opts['grains'] = {} load = Loader(module_dirs, opts, 'grain') grains = load.gen_grains() if 'grains' in opts: From 15e0ff5483cebc2882bb0cf36d07e83ef100c7b5 Mon Sep 17 00:00:00 2001 From: Evan Borgstrom <evan@fatbox.ca> Date: Sun, 26 Feb 2012 16:11:34 -0500 Subject: [PATCH 107/598] When using isintance to detect strings we should check for basestring I spent the past 30 min trying to debug a problem where when a command was published to the minion it was arriving as a string instead of a tuple. The cmd.run module was receiving: { 'arg': 'mycommand myarg1 myarg2' 'fun': 'cmd.run', 'jid': '20120226160155389058', 'ret': '', 'tgt': 'mytarget' } This was causing the following exception: TypeError: run() takes at most 3 arguments (20 given) I traced this down to the publish module checking for an instance of 'str' and splitting on ','. In my instance the problem was being caused because Django likes to use Unicode strings everywhere and the isintance was returning False. I've replaced all instances of str with basestring in all the isintance checks. --- salt/cli/caller.py | 2 +- salt/minion.py | 12 ++++++------ salt/modules/publish.py | 2 +- salt/modules/saltutil.py | 2 +- salt/modules/selinux.py | 2 +- salt/modules/state.py | 2 +- salt/output.py | 2 +- salt/state.py | 16 ++++++++-------- salt/states/file.py | 2 +- 9 files changed, 21 insertions(+), 21 deletions(-) diff --git a/salt/cli/caller.py b/salt/cli/caller.py index d980e8c48f1d..e80345786043 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -51,7 +51,7 @@ def call(self): sys.exit(1) if hasattr(self.minion.functions[fun], '__outputter__'): oput = self.minion.functions[fun].__outputter__ - if isinstance(oput, str): + if isinstance(oput, basestring): ret['out'] = oput return ret diff --git a/salt/minion.py b/salt/minion.py index 653781352edd..e237ffeed933 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -194,7 +194,7 @@ def _handle_decoded_payload(self, data): Override this method if you wish to handle the decoded data differently. ''' - if isinstance(data['fun'], str): + if isinstance(data['fun'], basestring): if data['fun'] == 'sys.reload_modules': self.functions, self.returners = self.__load_modules() @@ -233,7 +233,7 @@ def _thread_return(self, data): arg = eval(data['arg'][ind]) if isinstance(arg, bool): data['arg'][ind] = str(data['arg'][ind]) - elif isinstance(arg, (dict, int, list, str)): + elif isinstance(arg, (dict, int, list, basestring)): data['arg'][ind] = arg else: data['arg'][ind] = str(data['arg'][ind]) @@ -292,7 +292,7 @@ def _thread_multi_return(self, data): arg = eval(data['arg'][ind][index]) if isinstance(arg, bool): data['arg'][ind][index] = str(data['arg'][ind][index]) - elif isinstance(arg, (dict, int, list, str)): + elif isinstance(arg, (dict, int, list, basestring)): data['arg'][ind][index] = arg else: data['arg'][ind][index] = str(data['arg'][ind][index]) @@ -354,7 +354,7 @@ def _return_pub(self, ret, ret_cmd='_return'): try: if hasattr(self.functions[ret['fun']], '__outputter__'): oput = self.functions[ret['fun']].__outputter__ - if isinstance(oput, str): + if isinstance(oput, basestring): load['out'] = oput except KeyError: pass @@ -362,7 +362,7 @@ def _return_pub(self, ret, ret_cmd='_return'): data = self.serial.dumps(payload) socket.send(data) ret_val = self.serial.loads(socket.recv()) - if isinstance(ret_val, str) and not ret_val: + if isinstance(ret_val, basestring) and not ret_val: # The master AES key has changed, reauth self.authenticate() payload['load'] = self.crypticle.dumps(load) @@ -613,7 +613,7 @@ def compound_match(self, tgt): ''' Runs the compound target check ''' - if not isinstance(tgt, str): + if not isinstance(tgt, basestring): log.debug('Compound target received that is not a string') return False ref = {'G': 'grain', diff --git a/salt/modules/publish.py b/salt/modules/publish.py index 09178cbd40cc..95326a6f6712 100644 --- a/salt/modules/publish.py +++ b/salt/modules/publish.py @@ -48,7 +48,7 @@ def publish(tgt, fun, arg=None, expr_form='glob', returner='', timeout=5): if isinstance(ast.literal_eval(arg), dict): arg = [arg,] except: - if isinstance(arg, str): + if isinstance(arg, basestring): arg = arg.split(',') auth = salt.crypt.SAuth(__opts__) diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index 375eb28c4a6b..3bc2ec25ba97 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -19,7 +19,7 @@ def _sync(form, env): ''' Sync the given directory in the given environment ''' - if isinstance(env, str): + if isinstance(env, basestring): env = env.split(',') ret = [] remote = set() diff --git a/salt/modules/selinux.py b/salt/modules/selinux.py index e1c3be646e47..6836d5ca3752 100644 --- a/salt/modules/selinux.py +++ b/salt/modules/selinux.py @@ -43,7 +43,7 @@ def setenforce(mode): ''' Set the enforcing mode ''' - if isinstance(mode, str): + if isinstance(mode, basestring): if mode.lower() == 'enforcing': mode = '1' elif mode.lower() == 'permissive': diff --git a/salt/modules/state.py b/salt/modules/state.py index 346342ac94fb..97dbbe4e8cc4 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -89,7 +89,7 @@ def sls(mods, env='base'): salt '*' state.sls core,edit.vim dev ''' st_ = salt.state.HighState(__opts__) - if isinstance(mods, str): + if isinstance(mods, basestring): mods = mods.split(',') high, errors = st_.render_highstate({env: mods}) if errors: diff --git a/salt/output.py b/salt/output.py index 3419a11e733a..45425b318b84 100644 --- a/salt/output.py +++ b/salt/output.py @@ -97,7 +97,7 @@ def __call__(self, data, **kwargs): )) changes = ' Changes: ' for key in ret['changes']: - if isinstance(ret['changes'][key], str): + if isinstance(ret['changes'][key], basestring): changes += (key + ': ' + ret['changes'][key] + '\n ') elif isinstance(ret['changes'][key], dict): diff --git a/salt/state.py b/salt/state.py index 955dee8c3367..5a904c3f67fb 100644 --- a/salt/state.py +++ b/salt/state.py @@ -132,7 +132,7 @@ def load_modules(self, data=None): if isinstance(data, dict): if data.get('redirect', False): redirect = {} - if isinstance(data['redirect'], str): + if isinstance(data['redirect'], basestring): redirects = [{data['state']: data['redirect']}] elif isinstance(data['redirect'], list): redirects = data['redirect'] @@ -300,7 +300,7 @@ def verify_high(self, high): else: fun = 0 for arg in body[state]: - if isinstance(arg, str): + if isinstance(arg, basestring): fun += 1 elif isinstance(arg, dict): # The arg is a dict, if the arg is require or @@ -442,7 +442,7 @@ def compile_high_data(self, high): funcs = set() names = set() for arg in run: - if isinstance(arg, str): + if isinstance(arg, basestring): funcs.add(arg) continue if isinstance(arg, dict): @@ -494,8 +494,8 @@ def reconcile_extend(self, high): for arg in run: update = False for hind in range(len(high[name][state])): - if isinstance(arg, str) and \ - isinstance(high[name][state][hind], str): + if isinstance(arg, basestring) and \ + isinstance(high[name][state][hind], basestring): # replacing the function, replace the index high[name][state].pop(hind) high[name][state].insert(hind, arg) @@ -543,7 +543,7 @@ def compile_template(self, template, env='', sls=''): Take the path to a template and return the high data structure derived from the template. ''' - if not isinstance(template, str): + if not isinstance(template, basestring): return {} if not os.path.isfile(template): return {} @@ -899,12 +899,12 @@ def merge_tops(self, tops): for comp in ctop[env][tgt]: if isinstance(comp, dict): cmatches.append(comp) - if isinstance(comp, str): + if isinstance(comp, basestring): cstates.add(comp) for comp in top[env][tgt]: if isinstance(comp, dict): matches.append(comp) - if isinstance(comp, str): + if isinstance(comp, basestring): states.add(comp) top[env][tgt] = matches top[env][tgt].extend(list(states)) diff --git a/salt/states/file.py b/salt/states/file.py index 7be41063da69..5e7e1510e1ef 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -496,7 +496,7 @@ def managed(name, source = single_src source_hash = single_hash break - elif isinstance(single, str): + elif isinstance(single, basestring): if single in mfiles: source = single break From aa08f8a52bb99f4960d0b7a9057266784c72e31b Mon Sep 17 00:00:00 2001 From: pille <pille+github@struction.de> Date: Mon, 27 Feb 2012 11:44:48 +0100 Subject: [PATCH 108/598] updated ubuntu install doc currently there's no single up-to-date salt-package anymore. it has been split into pieces. wrt #711. --- doc/topics/installation/debian.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/topics/installation/debian.rst b/doc/topics/installation/debian.rst index efc9caccc189..1c693d7bf01e 100644 --- a/doc/topics/installation/debian.rst +++ b/doc/topics/installation/debian.rst @@ -11,7 +11,9 @@ for Lucid:: aptitude -y install python-software-properties add-apt-repository ppa:saltstack/salt aptitude update - aptitude install salt + aptitude install salt-master # on the master + aptitude install salt-minion # on the minion + aptitude install salt-syndic # instead of a slaved master Debian ------ From e6481b875f9e0bc26a7676ba4921eac32fb03f92 Mon Sep 17 00:00:00 2001 From: Evan Borgstrom <evan@fatbox.ca> Date: Mon, 27 Feb 2012 10:43:56 -0500 Subject: [PATCH 109/598] Fix mysql __virtual__ to correctly detect config It looks like when the following commit[1] was made it broke the mysql module, and I haven't noticed until now some how. My guess is that __opts__ used to look like __opts__['mysql']['default_file'] where as it now looks like __opts__['mysql.default_file'] [1] = https://github.com/saltstack/salt/commit/b7e7cc3cb1ede997d4af3a3ea2826d6f80ab2dcf --- salt/modules/mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index 95e12773774a..9b16b9ffd219 100755 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -27,7 +27,7 @@ def __virtual__(): ''' Only load this module if the mysql config is set ''' - if 'mysql' in __opts__: + if any(k.startswith('mysql.') for k in __opts__.keys()): return 'mysql' return False From a74c104785020d23914b01896f28ca578a641428 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 27 Feb 2012 13:48:48 -0700 Subject: [PATCH 110/598] Remove redundant code --- salt/minion.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 20280968db56..cce2ae33a5fe 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -117,21 +117,6 @@ def __load_modules(self): ''' Return the functions and the returners loaded up from the loader module ''' - pre_opts = {} - salt.config.load_config( - pre_opts, - self.opts['conf_file'], - 'SALT_MINION_CONFIG' - ) - if 'include' in pre_opts: - pre_opts = salt.config.include_config( - pre_opts, - self.opts['conf_file'] - ) - if 'grains' in pre_opts: - self.opts['grains'] = pre_opts['grains'] - else: - self.opts['grains'] = {} self.opts['grains'] = salt.loader.grains(self.opts) functions = salt.loader.minion_mods(self.opts) returners = salt.loader.returners(self.opts) From 71c391d7718509459a6055d3154bd793e1b9532b Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 27 Feb 2012 14:01:40 -0700 Subject: [PATCH 111/598] pidfile missing from cli struct --- salt/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/__init__.py b/salt/__init__.py index 4b0f52656f5a..cacee983704a 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -343,6 +343,7 @@ def __parse_cli(self): cli = {'daemon': options.daemon, 'minion_config': options.minion_config, 'master_config': options.master_config, + 'pidfile': options.pidfile, 'user': options.user} return cli From 2857276b186e5b85f0a6440845af04d9da7391e1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 27 Feb 2012 15:15:02 -0700 Subject: [PATCH 112/598] Add passign master's id to syndic return --- salt/minion.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index cce2ae33a5fe..044a74b9c060 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -325,7 +325,8 @@ def _return_pub(self, ret, ret_cmd='_return'): payload = {'enc': 'aes'} if ret_cmd == '_syndic_return': load = {'cmd': ret_cmd, - 'jid': ret['jid']} + 'jid': ret['jid'], + 'id': self.opts['id']} load['return'] = {} for key, value in ret.items(): if key == 'jid' or key == 'fun': From 34be8f2c4b759e4709cddd5779a604b0d88aa23d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 27 Feb 2012 15:19:28 -0700 Subject: [PATCH 113/598] Add wtag checks to master --- salt/master.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/salt/master.py b/salt/master.py index a240b11d7b79..b2f4a09c82ea 100644 --- a/salt/master.py +++ b/salt/master.py @@ -529,14 +529,35 @@ def _syndic_return(self, load): individual minions. ''' # Verify the load - if 'return' not in load or 'jid' not in load: + if 'return' not in load or 'jid' not in load or 'id' not in load: return None + # set the write flag + jid_dir = os.path.join(self.opts['cachedir'], 'jobs', load['jid']) + if not os.path.isdir(jid_dir): + log.error( + 'An inconsistency occurred, a job was received with a job id ' + 'that is not present on the master: %(jid)s', load + ) + return False + wtag = os.path.join(jid_dir, 'wtag_{0}'.format(load['id'])) + try: + open(wtag, 'w+').write('') + except IOError: + log.error( + ('Failed to commit the write tag for the syndic return,' + ' are permissions correct in the cache dir:' + ' {0}?').format(self.opts['cachedir']) + ) + return False + # Format individual return loads for key, item in load['return'].items(): ret = {'jid': load['jid'], 'id': key, 'return': item} self._return(ret) + if os.path.isfile(wtag): + os.remove(wtag) def minion_publish(self, clear_load): ''' From f6fd3cbd7ab7da30713dabfe0c2982542d6e8823 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 27 Feb 2012 15:19:44 -0700 Subject: [PATCH 114/598] Add wtag checks to the client --- salt/client.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/client.py b/salt/client.py index 9c41a39c713c..299da799a062 100644 --- a/salt/client.py +++ b/salt/client.py @@ -261,6 +261,7 @@ def get_returns(self, jid, minions, timeout=None): start = 999999999999 gstart = int(time.time()) ret = {} + wtag = os.path.join(jid_dir, 'wtag*') # Check to see if the jid is real, if not return the empty dict if not os.path.isdir(jid_dir): return ret @@ -280,9 +281,15 @@ def get_returns(self, jid, minions, timeout=None): pass if ret and start == 999999999999: start = int(time.time()) + if glob.glob(wtag) and not int(time.time()) > start + timeout + 1: + # The timeout +1 has not been reached and there is still a + # write tag for the syndic + continue if len(ret) >= len(minions): + # All Minions have returned return ret if int(time.time()) > start + timeout: + # The timeout has been reached return ret if int(time.time()) > gstart + timeout and not ret: # No minions have replied within the specified global timeout, From 4d586e3f6cebf5f056889d1b6c42ed5eb9da4a68 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 27 Feb 2012 15:34:06 -0700 Subject: [PATCH 115/598] more concise exception catch --- salt/master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/master.py b/salt/master.py index b2f4a09c82ea..a1a6f7239a8f 100644 --- a/salt/master.py +++ b/salt/master.py @@ -542,7 +542,7 @@ def _syndic_return(self, load): wtag = os.path.join(jid_dir, 'wtag_{0}'.format(load['id'])) try: open(wtag, 'w+').write('') - except IOError: + except (IOError, OSError): log.error( ('Failed to commit the write tag for the syndic return,' ' are permissions correct in the cache dir:' From 2f065648a1f6fb8f054e96d03c981d72aac02bf1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 27 Feb 2012 15:44:49 -0700 Subject: [PATCH 116/598] apply wtag checks to all cmd interfaces that need them --- salt/client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/client.py b/salt/client.py index 299da799a062..2d8e7ada1099 100644 --- a/salt/client.py +++ b/salt/client.py @@ -214,6 +214,7 @@ def get_iter_returns(self, jid, minions, timeout=None): start = 999999999999 gstart = int(time.time()) found = set() + wtag = os.path.join(jid_dir, 'wtag*') # Check to see if the jid is real, if not return the empty dict if not os.path.isdir(jid_dir): yield {} @@ -240,6 +241,10 @@ def get_iter_returns(self, jid, minions, timeout=None): yield ret if ret and start == 999999999999: start = int(time.time()) + if glob.glob(wtag) and not int(time.time()) > start + timeout + 1: + # The timeout +1 has not been reached and there is still a + # write tag for the syndic + continue if len(ret) >= len(minions): break if int(time.time()) > start + timeout: @@ -308,6 +313,7 @@ def get_full_returns(self, jid, minions, timeout=None): start = 999999999999 gstart = int(time.time()) ret = {} + wtag = os.path.join(jid_dir, 'wtag*') # Check to see if the jid is real, if not return the empty dict if not os.path.isdir(jid_dir): return ret @@ -331,6 +337,10 @@ def get_full_returns(self, jid, minions, timeout=None): pass if ret and start == 999999999999: start = int(time.time()) + if glob.glob(wtag) and not int(time.time()) > start + timeout + 1: + # The timeout +1 has not been reached and there is still a + # write tag for the syndic + continue if len(ret) >= len(minions): return ret if int(time.time()) > start + timeout: From 1e232110002adc0103d164fa436b61b799e8ddc0 Mon Sep 17 00:00:00 2001 From: Grier Johnson <grierj@gmail.com> Date: Mon, 27 Feb 2012 14:45:20 -0800 Subject: [PATCH 117/598] The upstream load has the key tgt_type and not expr_form, this makes for malformed loads --- salt/minion.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 044a74b9c060..920aae52cbde 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -493,15 +493,15 @@ def syndic_cmd(self, data): ''' Take the now clear load and forward it on to the client cmd ''' - # Set up default expr_form - if 'expr_form' not in data: - data['expr_form'] = 'glob' + # Set up default tgt_type + if 'tgt_type' not in data: + data['tgt_type'] = 'glob' # Send out the publication pub_data = self.pub( data['tgt'], data['fun'], data['arg'], - data['expr_form'], + data['tgt_type'], data['ret'], data['jid'], data['to'] From 57d417fe4176be221c6eb4b700a6940a47483f60 Mon Sep 17 00:00:00 2001 From: Erik Johnson <palehose@gmail.com> Date: Mon, 27 Feb 2012 16:51:04 -0600 Subject: [PATCH 118/598] simplify MAC address example by using network.hwaddr instead of parsing ifconfig with cmd.run --- doc/topics/tutorials/states_pt3.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/topics/tutorials/states_pt3.rst b/doc/topics/tutorials/states_pt3.rst index 62d93af542c2..8b3e7b532254 100644 --- a/doc/topics/tutorials/states_pt3.rst +++ b/doc/topics/tutorials/states_pt3.rst @@ -87,10 +87,10 @@ The Salt module functions are also made available in the template context as - group: {{ usr }} {% endfor %} -Below is another example that calls an arbitrary command in order to grab the -mac addr for eth0:: +Below is an example that uses the ``network.hwaddr`` function to retrieve the +MAC address for eth0: - salt['cmd.run']('ifconfig eth0 | grep HWaddr | cut -d" " -f10') + salt['network.hwaddr']('eth0') Advanced SLS module syntax ========================== From 14cde50cba7e805d267465603fddd6a38d3e78e0 Mon Sep 17 00:00:00 2001 From: Grier Johnson <grierj@gmail.com> Date: Mon, 27 Feb 2012 15:39:58 -0800 Subject: [PATCH 119/598] Enhanced Logging For Executing User Track the executing user and do some best effort work to figure out who it is based on environment variables. Pass the user in the payload and allow for logging of summary information about the job the user ran, but also allow for a raw dump of the user's target, arguments, etc. This aids in auditability of a user on a system trying to figure out what made a change and when. --- salt/client.py | 18 ++++++++++++++++++ salt/master.py | 5 +++++ salt/minion.py | 8 +++++--- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/salt/client.py b/salt/client.py index 2d8e7ada1099..d22622db7702 100644 --- a/salt/client.py +++ b/salt/client.py @@ -32,6 +32,7 @@ import glob import time import datetime +import getpass # Import zmq modules import zmq @@ -67,6 +68,7 @@ def __init__(self, c_path='/etc/salt/master'): self.opts = salt.config.master_config(c_path) self.serial = salt.payload.Serial(self.opts) self.key = self.__read_master_key() + self.salt_user = self.__get_user() def __read_master_key(self): ''' @@ -80,6 +82,20 @@ def __read_master_key(self): raise SaltClientError(('Problem reading the salt root key. Are' ' you root?')) + def __get_user(self): + ''' + Determine the current user running the salt command + ''' + user = getpass.getuser() + # if our user is root, look for other ways to figure out + # who we are + if user == 'root': + env_vars = ['SUDO_USER', 'USER', 'USERNAME'] + for evar in env_vars: + if evar in os.environ: + return os.environ[evar] + return user + def _check_glob_minions(self, expr): ''' Return the minions found by looking via globs @@ -448,6 +464,7 @@ def pub(self, tgt, fun, arg=(), expr_form='glob', tgt_type=expr_form, ret=ret, jid=jid, + user=self.salt_user, to=timeout) else: package = salt.payload.format_payload( @@ -459,6 +476,7 @@ def pub(self, tgt, fun, arg=(), expr_form='glob', key=self.key, tgt_type=expr_form, jid=jid, + user=self.salt_user, ret=ret) # Prep zmq context = zmq.Context() diff --git a/salt/master.py b/salt/master.py index a1a6f7239a8f..82148f069a8e 100644 --- a/salt/master.py +++ b/salt/master.py @@ -836,7 +836,12 @@ def publish(self, clear_load): 'tgt': clear_load['tgt'], 'jid': clear_load['jid'], 'ret': clear_load['ret'], + 'user': clear_load['user'], } + + log.info('{0[user]} published command {0[fun]} with jid {0[jid]}'.format(load)) + log.debug('Published command details {0}'.format(load)) + if 'tgt_type' in clear_load: load['tgt_type'] = clear_load['tgt_type'] if 'to' in clear_load: diff --git a/salt/minion.py b/salt/minion.py index 920aae52cbde..2316145d1392 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -159,7 +159,8 @@ def _handle_aes(self, load): # from returning a predictable exception #if data['fun'] not in self.functions: # return - log.debug('Executing command {0[fun]} with jid {0[jid]}'.format(data)) + log.info('{0[user]} executed command {0[fun]} with jid {0[jid]}'.format(data)) + log.debug('Command details {0}'.format(data)) self._handle_decoded_payload(data) def _handle_pub(self, load): @@ -471,8 +472,9 @@ def _handle_aes(self, load): or 'to' not in data or 'arg' not in data: return data['to'] = int(data['to']) - 1 - log.debug(('Executing syndic command {0[fun]} with jid {0[jid]}' - .format(data))) + log.info(('{0[user]} executed syndic command {0[fun]} with jid {0[jid]}' + .format(data))) + log.debug('Command details {0}'.format(data)) self._handle_decoded_payload(data) def _handle_decoded_payload(self, data): From f227c335b6c10146a7c9c9ed54895cfa48f771aa Mon Sep 17 00:00:00 2001 From: Christer Edwards <christer.edwards@gmail.com> Date: Mon, 27 Feb 2012 21:19:29 -0700 Subject: [PATCH 120/598] formatting and clarification cleanup in docs --- doc/topics/installation/freebsd.rst | 46 +++++++++++--------- doc/topics/tutorials/starting_states.rst | 54 ++++++++++++++---------- doc/topics/tutorials/states_pt2.rst | 22 +++++----- 3 files changed, 70 insertions(+), 52 deletions(-) diff --git a/doc/topics/installation/freebsd.rst b/doc/topics/installation/freebsd.rst index 043d90078c24..73b48be4a91a 100644 --- a/doc/topics/installation/freebsd.rst +++ b/doc/topics/installation/freebsd.rst @@ -33,9 +33,9 @@ Configuration In the sections below I'll outline configuration options for both the Salt Master and Salt Minions. -The Salt port installs two sample configuration files, salt/master.sample and -salt/minion.sample (these should be installed in /usr/local/etc/, unless you use a -different %%PREFIX%%). You'll need to copy these .sample files into place and +The Salt port installs two sample configuration files, ``salt/master.sample`` and +``salt/minion.sample`` (these should be installed in ``/usr/local/etc/``, unless you use a +different ``%%PREFIX%%``). You'll need to copy these .sample files into place and make a few edits. First, copy them into place as seen here:: cp /usr/local/etc/salt/master.sample /usr/local/etc/salt/master @@ -59,7 +59,9 @@ configuration paths. By default the Salt master listens on ports 4505 and 4506 on all interfaces (0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the -"interface" directive as seen here:: +"interface" directive as seen here. + +.. code-block:: diff - #interface: 0.0.0.0 + interface: 10.0.0.1 @@ -68,7 +70,9 @@ By default the Salt master listens on ports 4505 and 4506 on all interfaces Last but not least you'll need to activate the Salt Master in your rc.conf file. Using your favorite editor, open /etc/rc.conf or /etc/rc.conf.local and -add this line:: +add this line. + +.. code-block:: diff + salt_master_enable="YES" @@ -98,7 +102,9 @@ this you likely can do without any minion configuration at all. If you are not able to update DNS, you'll simply need to update one entry in the configuration file. Using your favorite editor, open the minion -configuration file and update the "master" entry as seen here:: +configuration file and update the "master" entry as seen here. + +.. code-block:: diff - #master: salt + master: 10.0.0.1 @@ -111,14 +117,16 @@ configuration options are covered in another chapter. Before you're able to start the Salt Minion you'll need to update your rc.conf file. Using your favorite editor open /etc/rc.conf or /etc/rc.conf.local and -add this line:: +add this line. + +.. code-block:: diff + salt_minion_enable="YES" Once you've completed all of these steps you're ready to start your Salt Minion. The Salt port installs an rc script which should be used to manage your Salt Minion. You should be able to start your Salt Minion now using the command -seen here:: +seen here. :: service salt_minion start @@ -149,10 +157,10 @@ list the keys known to the Salt Master:: [root@master ~]# salt-key -L Unaccepted Keys: - avon - bodie - bubbles - marlo + alpha + bravo + charlie + delta Accepted Keys: This example shows that the Salt Master is aware of four Minions, but none of @@ -163,10 +171,10 @@ controlled by the Master, again use the ``salt-key`` command:: [root@master ~]# salt-key -L Unaccepted Keys: Accepted Keys: - avon - bodie - bubbles - marlo + alpha + bravo + charlie + delta The ``salt-key`` command allows for signing keys individually or in bulk. The example above, using ``-A`` bulk-accepts all pending keys. To accept keys @@ -181,10 +189,10 @@ Whether you have a few or a few-dozen, Salt can help you manage them easily! For final verification, send a test function from your Salt Master to your minions. If all of your minions are properly communicating with your Master, you should "True" responses from each of them. See the example below to send -the ``test.ping`` remote command:: +the ``test.ping`` remote command. :: - [root@master ~]# salt '*' test.ping - {'avon': True} + [root@master ~]# salt 'alpha' test.ping + {'alpha': True} Where Do I Go From Here ======================== diff --git a/doc/topics/tutorials/starting_states.rst b/doc/topics/tutorials/starting_states.rst index f3d922a1c67a..b20572bf3f65 100644 --- a/doc/topics/tutorials/starting_states.rst +++ b/doc/topics/tutorials/starting_states.rst @@ -19,7 +19,7 @@ is just a data structure under the hood. While understanding that the SLS is just a data structure is not at all critical to understand to make use Salt States, it should help bolster the understanding of where the real power is. -SLS files are therefore, in reality, just dictionaries, lists strings and +SLS files are therefore, in reality, just dictionaries, lists, strings and numbers. By using this approach Salt can be much more flexible, and as someone writes more SLS files it becomes clear exactly what is being written. The result is a system that is easy to understand, yet grows with the needs of the admin @@ -38,6 +38,7 @@ serialization formats available - YAML. A typical, small SLS file will often look like this in YAML: .. code-block:: yaml + :linenos: apache: pkg: @@ -74,6 +75,7 @@ need to be added. The apache configuration file will most likely be managed, and a user and group may need to be set up. .. code-block:: yaml + :linenos: apache: pkg: @@ -138,10 +140,10 @@ The SLS files are laid out in a directory on the salt master. Files are laid out as just files, an sls is just a file and files to download are just files. The apache example would be laid out in the root of the salt file server like -this: +this: :: -/apache/init.sls -/apache/httpd.conf + /apache/init.sls + /apache/httpd.conf So the httpd.conf is just a file in the apache directory, and is referenced directly. @@ -149,14 +151,15 @@ directly. But with more than a single SLS file, more components can be added to the toolkit, consider this ssh example: -``/ssh/init.sls`` +``/ssh/init.sls:`` .. code-block:: yaml - + :linenos: + openssh-client: pkg: - installed - + /etc/ssh/ssh_config file: - managed @@ -167,9 +170,10 @@ toolkit, consider this ssh example: - require: - pkg: openssh-client -``/ssh/server.sls`` +``ssh/server.sls:`` .. code-block:: yaml + :linenos: include: - ssh @@ -206,15 +210,15 @@ toolkit, consider this ssh example: - require: - pkg: openssh-server -Now our State Tree looks like this: +Now our State Tree looks like this: :: -/apache/init.sls -/apache/httpd.conf -/ssh/init.sls -/ssh/server.sls -/ssh/banner -/ssh/ssh_config -/ssh/sshd_config + /apache/init.sls + /apache/httpd.conf + /ssh/init.sls + /ssh/server.sls + /ssh/banner + /ssh/ssh_config + /ssh/sshd_config This example now introduces the ``include`` statement. The include statement includes another SLS file so that components found in it can be required, @@ -233,9 +237,10 @@ needs to be placed. These examples will add more watchers to apache and change the ssh banner. -``/ssh/custom-server.sls`` +``/ssh/custom-server.sls:`` .. code-block:: yaml + :linenos: include: - ssh.server @@ -245,9 +250,10 @@ These examples will add more watchers to apache and change the ssh banner. file: - source: salt://ssh/custom-banner -``/python/mod_python.sls`` +``/python/mod_python.sls:`` .. code-block:: yaml + :linenos: include: - apache @@ -281,7 +287,7 @@ with YAML. Salt defaults to YAML because it is very straightforward and easy to learn and use. But the SLS files can be rendered from almost any imaginable medium, so long as a renderer module is provided. -The default rendering system is the ``yaml_jinja`` renderer. The +The default rendering system is the ``yaml_jinja`` renderer. The ``yaml_jinja`` renderer will first pass the template through the jinja templating system, and then through the YAML parser. The benefit here is that full programming constructs are available when creating SLS files. @@ -305,9 +311,10 @@ available, ``salt`` and ``grains``. The salt object allows for any salt function to be called from within the template, and grains allows for the grains to be accessed from within the template. A few examples are in order: -``/apache/init.sls`` +``/apache/init.sls:`` .. code-block:: yaml + :linenos: apache: pkg: @@ -352,9 +359,10 @@ Red Hat, then the name of the apache package and service needs to be httpd. A more aggressive way to use Jinja can be found here, in a module to set up a MooseFS distributed filesystem chunkserver: -``/moosefs/chunk.sls`` +``/moosefs/chunk.sls:`` .. code-block:: yaml + :linenos: include: - moosefs @@ -427,9 +435,10 @@ but a SLS file set to use another renderer can be easily added to the tree. This example shows a very basic python SLS file: -``/python/django.sls`` +``/python/django.sls:`` .. code-block:: python + :linenos: #!py @@ -449,6 +458,7 @@ must be a Salt friendly data structure, or better known as a Salt This python example would look like this if it were written in YAML: .. code-block:: yaml + :linenos: include: - python diff --git a/doc/topics/tutorials/states_pt2.rst b/doc/topics/tutorials/states_pt2.rst index 4a43811a2e44..24b3c51d4642 100644 --- a/doc/topics/tutorials/states_pt2.rst +++ b/doc/topics/tutorials/states_pt2.rst @@ -40,9 +40,9 @@ directory named ``webserver`` and moving and renaming ``webserver.sls`` to :: - |- top.sls - `- webserver/ - `- init.sls + |- top.sls + `- webserver/ + `- init.sls .. admonition:: Organizing SLS modules @@ -65,7 +65,7 @@ installed and running. Include the following at the bottom of your .. code-block:: yaml :linenos: - :emphasize-lines: 6,11 + :emphasize-lines: 7,11 apache: pkg: @@ -80,25 +80,25 @@ installed and running. Include the following at the bottom of your - require: # requisite declaration - pkg: apache # requisite reference -Again in **line 1** is the :term:`ID declaration`. In this example it is the +**line 7** is the :term:`ID declaration`. In this example it is the location we want to install our custom HTML file. (**Note:** the default location that Apache serves may differ from the above on your OS or distro. ``/srv/www`` could also be a likely place to look.) -**Line 2** the :term:`state declaration`. This example uses the Salt :mod:`file +**Line 8** the :term:`state declaration`. This example uses the Salt :mod:`file state <salt.states.file>`. -**Line 3** is the :term:`function declaration`. The :func:`managed function +**Line 9** is the :term:`function declaration`. The :func:`managed function <salt.states.file.managed>` will download a file from the master and install it in the location specified. -**Line 4** is a :term:`function arg declaration` which, in this example, passes +**Line 10** is a :term:`function arg declaration` which, in this example, passes the ``source`` argument to the :func:`managed function -<salt.states.file.managed>`. +<salt.states.file.managed>`. -**Line 5** is a :term:`requisite declaration`. +**Line 11** is a :term:`requisite declaration`. -**Line 6** is a :term:`requisite reference` which refers to a state and an ID. +**Line 12** is a :term:`requisite reference` which refers to a state and an ID. In this example, it is referring to the ``ID declaration`` from our example in :doc:`part 1 <states_pt1>`. This declaration tells Salt not to install the HTML file until Apache is installed. From e62e7cceb48e7b6e6a92b27ac9ebdc4aad220c71 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 27 Feb 2012 23:37:59 -0700 Subject: [PATCH 121/598] Fix compatability bug in user data transfer --- salt/master.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/salt/master.py b/salt/master.py index 82148f069a8e..62721e4c5eb7 100644 --- a/salt/master.py +++ b/salt/master.py @@ -827,19 +827,30 @@ def publish(self, clear_load): if not os.path.isdir(jid_dir): os.makedirs(jid_dir) # Save the invocation information - self.serial.dump(clear_load, open(os.path.join(jid_dir, '.load.p'), 'w+')) + self.serial.dump( + clear_load, + open(os.path.join(jid_dir, '.load.p'), 'w+') + ) # Set up the payload payload = {'enc': 'aes'} + # Altering the contents of the publish load is serious!! Changes here + # break compatibility with minion/master versions and even tiny + # additions can have serious implications on the performance of the + # publish commands. + # + # In short, check with Thomas Hatch before you even think about + # touching this stuff, we can probably do what you want to do another + # way that won't have a negative impact. load = { 'fun': clear_load['fun'], 'arg': clear_load['arg'], 'tgt': clear_load['tgt'], 'jid': clear_load['jid'], 'ret': clear_load['ret'], - 'user': clear_load['user'], } - log.info('{0[user]} published command {0[fun]} with jid {0[jid]}'.format(load)) + log.info(('User {0[user]} Published command {0[fun]} with jid' + ' {0[jid]}').format(clear_load)) log.debug('Published command details {0}'.format(load)) if 'tgt_type' in clear_load: From a19dfbc12dffc2d9651a79ad81b1c781d2a7ccbc Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 27 Feb 2012 23:39:07 -0700 Subject: [PATCH 122/598] fix comparison issue in compond matcher --- salt/minion.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 2316145d1392..b417d04439c9 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -159,7 +159,7 @@ def _handle_aes(self, load): # from returning a predictable exception #if data['fun'] not in self.functions: # return - log.info('{0[user]} executed command {0[fun]} with jid {0[jid]}'.format(data)) + log.info('Executing command {0[fun]} with jid {0[jid]}'.format(data)) log.debug('Command details {0}'.format(data)) self._handle_decoded_payload(data) @@ -472,8 +472,8 @@ def _handle_aes(self, load): or 'to' not in data or 'arg' not in data: return data['to'] = int(data['to']) - 1 - log.info(('{0[user]} executed syndic command {0[fun]} with jid {0[jid]}' - .format(data))) + log.debug(('Executing syndic command {0[fun]} with jid {0[jid]}' + .format(data))) log.debug('Command details {0}'.format(data)) self._handle_decoded_payload(data) @@ -608,6 +608,7 @@ def compound_match(self, tgt): 'L': 'list', 'E': 'pcre'} results = [] + opers = ['and', 'or', 'not'] for match in tgt.split(): # Attach the boolean operator if match == 'and': @@ -622,7 +623,7 @@ def compound_match(self, tgt): # If we are here then it is not a boolean operator, check if the # last member of the result list is a boolean, if no, append and if results: - if results[-1] != 'and' or results[-1] != 'or' or results[-1] != 'not': + if results[-1] not in opers: results.append('and') if match[1] == '@': comps = match.split('@') From 04cc3863b35660b62d6fd825d72eae22849186f1 Mon Sep 17 00:00:00 2001 From: Gordon McAllister <gordon.mcallister@gmail.com> Date: Mon, 27 Feb 2012 23:15:14 -0800 Subject: [PATCH 123/598] Fixes for compound matching Previously booleans were appended to the results list unnecessarily in some cases and all matchers were evaluated twice. --- salt/minion.py | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index b417d04439c9..d5dc544a0452 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -610,21 +610,8 @@ def compound_match(self, tgt): results = [] opers = ['and', 'or', 'not'] for match in tgt.split(): - # Attach the boolean operator - if match == 'and': - results.append('and') - continue - elif match == 'or': - results.append('or') - continue - elif match == 'not': - results.append('not') - continue - # If we are here then it is not a boolean operator, check if the - # last member of the result list is a boolean, if no, append and - if results: - if results[-1] not in opers: - results.append('and') + # Try to match tokens from the compound target, first by using + # the 'G, X, L, E' matcher types, then by hostname glob. if match[1] == '@': comps = match.split('@') matcher = ref.get(comps[0]) @@ -637,14 +624,12 @@ def compound_match(self, tgt): '{0}_match'.format(matcher) )('@'.join(comps[1:])) )) + elif match in opers: + # We didn't match a target, so append a boolean operator + results.append(match) else: - results.append( - str(getattr( - self, - '{0}_match'.format(matcher) - )('@'.join(comps[1:])) - )) - + # Try to match by glob + results.append(str(self.glob_match(match))) return eval(' '.join(results)) def nodegroup_match(self, tgt, nodegroups): From ab7c16b92be909a8c9188463d6c23b457c567193 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 28 Feb 2012 00:20:14 -0700 Subject: [PATCH 124/598] move log forward so that data is more accurate --- salt/master.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/master.py b/salt/master.py index 62721e4c5eb7..507ce3fd9c7f 100644 --- a/salt/master.py +++ b/salt/master.py @@ -849,14 +849,15 @@ def publish(self, clear_load): 'ret': clear_load['ret'], } - log.info(('User {0[user]} Published command {0[fun]} with jid' - ' {0[jid]}').format(clear_load)) - log.debug('Published command details {0}'.format(load)) - if 'tgt_type' in clear_load: load['tgt_type'] = clear_load['tgt_type'] if 'to' in clear_load: load['to'] = clear_load['to'] + + log.info(('User {0[user]} Published command {0[fun]} with jid' + ' {0[jid]}').format(clear_load)) + log.debug('Published command details {0}'.format(load)) + payload['load'] = self.crypticle.dumps(load) # Send 0MQ to the publisher context = zmq.Context(1) From 3e3324e5d0db07b6bcf6b1a91392f14f1ba4c751 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 28 Feb 2012 00:32:24 -0700 Subject: [PATCH 125/598] fix unspecified glob match in compound matcher --- salt/minion.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index b417d04439c9..ab0b7d967591 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -621,11 +621,14 @@ def compound_match(self, tgt): results.append('not') continue # If we are here then it is not a boolean operator, check if the - # last member of the result list is a boolean, if no, append and + # last member of the result list is a boolean, if no, append "and" if results: + # Results exist, check if we need to add an and because 2 + # match statements are back to back if results[-1] not in opers: results.append('and') - if match[1] == '@': + if '@' in match and match[1] == '@': + # If the match is explicitly defined then evaluate the matcher comps = match.split('@') matcher = ref.get(comps[0]) if not matcher: @@ -638,12 +641,8 @@ def compound_match(self, tgt): )('@'.join(comps[1:])) )) else: - results.append( - str(getattr( - self, - '{0}_match'.format(matcher) - )('@'.join(comps[1:])) - )) + # The match is not explicitely defined, evaluate it as a glob + results.append(str(self.glob_match(match))) return eval(' '.join(results)) From 3c162240d7f04a8e504a0c5bb4f9a216b9d341de Mon Sep 17 00:00:00 2001 From: Devon Stewart <blast@hardchee.se> Date: Tue, 28 Feb 2012 02:06:33 -0800 Subject: [PATCH 126/598] Adding debconf module Allows getting all selections, getting a single package, and setting the answer to a single question for a single package. --- salt/modules/debconf.py | 100 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 salt/modules/debconf.py diff --git a/salt/modules/debconf.py b/salt/modules/debconf.py new file mode 100644 index 000000000000..ccf7ebb94230 --- /dev/null +++ b/salt/modules/debconf.py @@ -0,0 +1,100 @@ +''' +Support for Debconf +''' + +import os +import re +import tempfile + + +def _unpack_lines(out): + rexp = ('(?ms)' + '^(?P<package>[^#]\S+)[\t ]+' + '(?P<question>\S+)[\t ]+' + '(?P<type>\S+)[\t ]+' + '(?P<value>[^\n]*)$') + lines = re.findall(rexp, out) + return lines + + +def __virtual__(): + ''' + Confirm this module is on a Debian based system + ''' + + return 'debconf' if __grains__['os'] in ['Debian', 'Ubuntu'] else False + + +def get_selections(fetchempty=True): + ''' + Answers to debconf questions for all packages in the following format:: + + {'package': [['question', 'type', 'value'], ...]} + + CLI Example:: + + salt '*' debconf.get_selections + ''' + selections = {} + cmd = 'debconf-get-selections' + + out = __salt__['cmd.run_stdout'](cmd) + + lines = _unpack_lines(out) + + for line in lines: + package, question, type, value = line + if fetchempty or value: + (selections + .setdefault(package, []) + .append([question, type, value])) + + return selections + + +def show(name): + ''' + Answers to debconf questions for a package in the following format:: + + [['question', 'type', 'value'], ...] + + If debconf doesn't know about a package, we return None. + + CLI Example:: + + salt '*' debconf.show <package name> + ''' + + result = None + + selections = get_selections() + + result = selections.get(name) + return result + + +def set(package, question, type, value, *extra): + ''' + Set answers to debconf questions for a package. + + CLI Example:: + + salt '*' debconf.set <package> <question> <type> <value> [<value> ...] + ''' + + if extra: + value = ' '.join((value,) + tuple(extra)) + + fd, fname = tempfile.mkstemp(prefix="salt-") + + line = "{0} {1} {2} {3}".format(package, question, type, value) + os.write(fd, line) + os.close(fd) + + cmd = 'debconf-set-selections {0}'.format(fname) + + out = __salt__['cmd.run_stdout'](cmd) + + os.unlink(fname) + + return True From a3a37556693a854282cbbe24acb1a6e6b5a734d3 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 28 Feb 2012 11:01:08 -0700 Subject: [PATCH 127/598] Add client side changes for new grain-pcre matcher --- salt/cli/__init__.py | 11 +++++++++++ salt/client.py | 1 + 2 files changed, 12 insertions(+) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 31ccabc8e77b..d7a0a0d37069 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -69,6 +69,14 @@ def __parse(self): default=False, dest='grain', action='store_true', + help=('Instead of using shell globs to evaluate the target ' + 'use a grain value to identify targets, the syntax ' + 'for the target is the grain key followed by a glob' + 'expression:\n"os:Arch*"')) + parser.add_option('--grain-pcre', + default=False, + dest='grain_pcre', + action='store_true', help=('Instead of using shell globs to evaluate the target ' 'use a grain value to identify targets, the syntax ' 'for the target is the grain key followed by a pcre ' @@ -158,6 +166,7 @@ def __parse(self): opts['pcre'] = options.pcre opts['list'] = options.list_ opts['grain'] = options.grain + opts['grain_pcre'] = options.grain_pcre opts['exsel'] = options.exsel opts['nodegroup'] = options.nodegroup opts['compound'] = options.compound @@ -248,6 +257,8 @@ def run(self): args.append('list') elif self.opts['grain']: args.append('grain') + elif self.opts['grain_pcre']: + args.append('grain_pcre') elif self.opts['exsel']: args.append('exsel') elif self.opts['nodegroup']: diff --git a/salt/client.py b/salt/client.py index d22622db7702..335dc4b496be 100644 --- a/salt/client.py +++ b/salt/client.py @@ -406,6 +406,7 @@ def check_minions(self, expr, expr_form='glob'): 'pcre': self._check_pcre_minions, 'list': self._check_list_minions, 'grain': self._check_grain_minions, + 'grain_pcre': self._check_grain_minions, 'exsel': self._check_grain_minions, 'compound': self._check_grain_minions, }[expr_form](expr) From aedd16cb70e1b757def015afdbbc6efa86a46b60 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 28 Feb 2012 11:02:10 -0700 Subject: [PATCH 128/598] Add minion side grain matcher changes --- salt/minion.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index 0d07bc6bb229..391fc505b72d 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -571,7 +571,29 @@ def list_match(self, tgt): def grain_match(self, tgt): ''' - Reads in the grains regular expression match + Reads in the grains glob match + ''' + comps = tgt.split(':') + if len(comps) < 2: + log.error('Got insufficient arguments for grains from master') + return False + if comps[0] not in self.opts['grains']: + log.error('Got unknown grain from master: {0}'.format(comps[0])) + return False + if isinstance(self.opts['grains'][comps[0]], list): + # We are matching a single component to a single list member + for member in self.opts['grains'][comps[0]]: + if fnmatch.fnmatch(str(member), comps[1]): + return True + return False + return bool(fnmatch.fnmatch( + str(self.opts['grains'][comps[0]]), + comps[1], + )) + + def grain_pcre_match(self, tgt): + ''' + Matches a grain based on regex ''' comps = tgt.split(':') if len(comps) < 2: From 53c9b29a7a200dd733bc5230ecc9d0f2a1ebbca1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 28 Feb 2012 11:27:02 -0700 Subject: [PATCH 129/598] add grain_pcre tracking to salt-cp --- salt/cli/__init__.py | 11 ++++++++++- salt/cli/cp.py | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index d7a0a0d37069..945be87cf416 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -384,7 +384,15 @@ def __parse(self): action='store_true', help=('Instead of using shell globs to evaluate the target ' 'use a grain value to identify targets, the syntax ' - 'for the target is the grains key followed by a pcre ' + 'for the target is the grain key followed by a glob' + 'expression:\n"os:Arch*"')) + parser.add_option('--grain-pcre', + default=False, + dest='grain_pcre', + action='store_true', + help=('Instead of using shell globs to evaluate the target ' + 'use a grain value to identify targets, the syntax ' + 'for the target is the grain key followed by a pcre ' 'regular expression:\n"os:Arch.*"')) parser.add_option('-N', '--nodegroup', @@ -410,6 +418,7 @@ def __parse(self): opts['pcre'] = options.pcre opts['list'] = options.list_ opts['grain'] = options.grain + opts['grain_pcre'] = options.grain_pcre opts['nodegroup'] = options.nodegroup opts['conf_file'] = options.conf_file diff --git a/salt/cli/cp.py b/salt/cli/cp.py index 8a9316aa8610..e82f219f42ab 100644 --- a/salt/cli/cp.py +++ b/salt/cli/cp.py @@ -74,6 +74,8 @@ def run(self): args.append('list') elif self.opts['grain']: args.append('grain') + elif self.opts['grain_pcre']: + args.append('grain_pcre') elif self.opts['nodegroup']: args.append('nodegroup') From f6794b485cdfbb957eaddbc5a23bf29a322e3d6c Mon Sep 17 00:00:00 2001 From: Devon Stewart <blast@hardchee.se> Date: Tue, 28 Feb 2012 10:37:08 -0800 Subject: [PATCH 130/598] Adding set_file to debconf module Allows for a more natural interface to debconf --- salt/modules/debconf.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/salt/modules/debconf.py b/salt/modules/debconf.py index ccf7ebb94230..9fe6547c9a44 100644 --- a/salt/modules/debconf.py +++ b/salt/modules/debconf.py @@ -72,6 +72,10 @@ def show(name): result = selections.get(name) return result +def _set_file(path): + cmd = 'debconf-set-selections {0}'.format(path) + + out = __salt__['cmd.run_stdout'](cmd) def set(package, question, type, value, *extra): ''' @@ -91,10 +95,26 @@ def set(package, question, type, value, *extra): os.write(fd, line) os.close(fd) - cmd = 'debconf-set-selections {0}'.format(fname) - - out = __salt__['cmd.run_stdout'](cmd) + _set_file(fname) os.unlink(fname) return True + +def set_file(path): + ''' + Set answers to debconf questions from a file. + + CLI Example:: + + salt '*' debconf.set_file salt://pathto/pkg.selections + ''' + + r = False + + path = __salt__['cp.cache_file'](path) + if path: + _set_file(path) + r = True + + return r From b39f4a862af79a48d757354846f50868a0993037 Mon Sep 17 00:00:00 2001 From: Devon Stewart <blast@hardchee.se> Date: Tue, 28 Feb 2012 11:44:37 -0800 Subject: [PATCH 131/598] Adding debconf support to apt --- salt/modules/apt.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/modules/apt.py b/salt/modules/apt.py index 6d48123449a6..9a9f01a90963 100644 --- a/salt/modules/apt.py +++ b/salt/modules/apt.py @@ -77,7 +77,8 @@ def refresh_db(): return servers -def install(pkg, refresh=False, repo='', skip_verify=False, **kwargs): +def install(pkg, refresh=False, repo='', skip_verify=False, + debconf=None, **kwargs): ''' Install the passed package @@ -90,6 +91,9 @@ def install(pkg, refresh=False, repo='', skip_verify=False, **kwargs): (e.g., ``apt-get -t unstable install somepackage``) skip_verify : False Skip the GPG verification check (e.g., ``--allow-unauthenticated``) + debconf : None + Provide the path to a debconf answers file, processed before + installation. Return a dict containing the new package names and versions:: @@ -103,6 +107,9 @@ def install(pkg, refresh=False, repo='', skip_verify=False, **kwargs): if refresh: refresh_db() + if debconf: + __salt__['debconf.set_file'](debconf) + ret_pkgs = {} old_pkgs = list_pkgs() From 679a3843b198adad6102c1dc882f6a6f70252358 Mon Sep 17 00:00:00 2001 From: Grier Johnson <grierj@gmail.com> Date: Tue, 28 Feb 2012 14:31:34 -0800 Subject: [PATCH 132/598] Move user logging and passing optional Only pass the user in payload if it is not root and not the specified user from the configuration file. Those two users are implied to be the executor if there's no specific user. Log statements referencing the user have been wrapped in statements testing for the existence of the user key in the payload, otherwise log messages fall back to a non-user log message (it will continue to do this for root). This preserves backwards compatibility for masters that would never pass the user as well --- salt/client.py | 49 +++++++++++++++++++++++++------------------------ salt/master.py | 8 ++++++-- salt/minion.py | 17 +++++++++++++---- 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/salt/client.py b/salt/client.py index 335dc4b496be..8976a0bb38d4 100644 --- a/salt/client.py +++ b/salt/client.py @@ -94,6 +94,11 @@ def __get_user(self): for evar in env_vars: if evar in os.environ: return os.environ[evar] + return None + # If the running user is just the specified user in the + # conf file, don't pass the user as it's implied. + elif user == self.opts['user']: + return None return user def _check_glob_minions(self, expr): @@ -454,31 +459,27 @@ def pub(self, tgt, fun, arg=(), expr_form='glob', if not minions: return {'jid': '', 'minions': minions} + + # Generate the standard keyword args to feed to format_payload + payload_kwargs = { 'cmd': 'publish', + 'tgt': tgt, + 'fun': fun, + 'arg': arg, + 'key': self.key, + 'tgt_type': expr_form, + 'ret': ret, + 'jid': jid } + + # If we have a salt user, add it to the payload + if self.salt_user: + payload_kwargs['user'] = self.salt_user + + # If we're a syndication master, pass the timeout if self.opts['order_masters']: - package = salt.payload.format_payload( - 'clear', - cmd='publish', - tgt=tgt, - fun=fun, - arg=arg, - key=self.key, - tgt_type=expr_form, - ret=ret, - jid=jid, - user=self.salt_user, - to=timeout) - else: - package = salt.payload.format_payload( - 'clear', - cmd='publish', - tgt=tgt, - fun=fun, - arg=arg, - key=self.key, - tgt_type=expr_form, - jid=jid, - user=self.salt_user, - ret=ret) + payload_kwargs['to'] = timeout + + package = salt.payload.format_payload( 'clear', **payload_kwargs) + # Prep zmq context = zmq.Context() socket = context.socket(zmq.REQ) diff --git a/salt/master.py b/salt/master.py index 507ce3fd9c7f..a7f1e9ecfe5c 100644 --- a/salt/master.py +++ b/salt/master.py @@ -854,8 +854,12 @@ def publish(self, clear_load): if 'to' in clear_load: load['to'] = clear_load['to'] - log.info(('User {0[user]} Published command {0[fun]} with jid' - ' {0[jid]}').format(clear_load)) + if 'user' in clear_load: + log.info(('User {0[user]} Published command {0[fun]} with jid' + ' {0[jid]}').format(clear_load)) + else: + log.info(('Published command {0[fun]} with jid' + ' {0[jid]}').format(clear_load)) log.debug('Published command details {0}'.format(load)) payload['load'] = self.crypticle.dumps(load) diff --git a/salt/minion.py b/salt/minion.py index 391fc505b72d..bcc5a8253630 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -159,7 +159,12 @@ def _handle_aes(self, load): # from returning a predictable exception #if data['fun'] not in self.functions: # return - log.info('Executing command {0[fun]} with jid {0[jid]}'.format(data)) + if 'user' in data: + log.info(('User {0[user]} Executing command {0[fun]} with jid ' + '{0[jid]}'.format(data))) + else: + log.info(('Executing command {0[fun]} with jid {0[jid]}' + .format(data))) log.debug('Command details {0}'.format(data)) self._handle_decoded_payload(data) @@ -472,9 +477,13 @@ def _handle_aes(self, load): or 'to' not in data or 'arg' not in data: return data['to'] = int(data['to']) - 1 - log.debug(('Executing syndic command {0[fun]} with jid {0[jid]}' - .format(data))) - log.debug('Command details {0}'.format(data)) + if 'user' in data: + log.debug(('User {0[user]} Executing syndic command {0[fun]} with ' + 'jid {0[jid]}'.format(data))) + else: + log.debug(('Executing syndic command {0[fun]} with jid {0[jid]}' + .format(data))) + log.debug('Command details: {0}'.format(data)) self._handle_decoded_payload(data) def _handle_decoded_payload(self, data): From de8011adb2fb35497c0d081d8aba6c37d0c4467c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 28 Feb 2012 16:29:21 -0700 Subject: [PATCH 133/598] Allow states to cleanly accept **kwargs This addition makes it so that ALL data in the low state chunk is passed to the state function via **kwargs. This means that by passing a **kwargs from states all the back to a module function will allow for very transparent additions of arguments to states that accept **kwargs --- salt/state.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 8e1becc8eb79..c4e08f513d93 100644 --- a/salt/state.py +++ b/salt/state.py @@ -405,6 +405,14 @@ def format_call(self, data): arglen = len(aspec[0]) if isinstance(aspec[3], tuple): deflen = len(aspec[3]) + if aspec[2]: + # This state accepts kwargs + ret['kwargs'] = {} + for key in data: + # Passing kwargs the conflict with args == stack trace + if key in aspec[0]: + continue + ret['kwargs'][key] = data[key] kwargs = {} for ind in range(arglen - 1, 0, -1): minus = arglen - ind @@ -573,7 +581,10 @@ def call(self, data): if 'provider' in data: self.load_modules(data) cdata = self.format_call(data) - ret = self.states[cdata['full']](*cdata['args']) + if 'kwargs' in cdata: + ret = self.states[cdata['full']](*cdata['args'], **cdata['kwargs']) + else: + ret = self.states[cdata['full']](*cdata['args']) ret['__run_num__'] = self.__run_num self.__run_num += 1 format_log(ret) From cbe25bf64bcc722baced72d57032ff8e8934b997 Mon Sep 17 00:00:00 2001 From: Devon Stewart <blast@hardchee.se> Date: Tue, 28 Feb 2012 15:59:10 -0800 Subject: [PATCH 134/598] Adding **kwargs to pkg installed and latest --- salt/states/pkg.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index e485a4913c95..53dbd6942568 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -26,7 +26,7 @@ def __gen_rtag(): return os.path.join(__opts__['cachedir'], 'pkg_refresh') -def installed(name, version=None, refresh=False, repo='', skip_verify=False): +def installed(name, version=None, refresh=False, repo='', skip_verify=False, **kwargs): ''' Verify that the package is installed, and only that it is installed. This state will not upgrade an existing package and only verify that it is @@ -66,14 +66,16 @@ def installed(name, version=None, refresh=False, repo='', skip_verify=False): True, version=version, repo=repo, - skip_verify=skip_verify) + skip_verify=skip_verify, + **kwargs) if os.path.isfile(rtag): os.remove(rtag) else: changes = __salt__['pkg.install'](name, version=version, repo=repo, - skip_verify=skip_verify) + skip_verify=skip_verify, + **kwargs) if not changes: return {'name': name, 'changes': changes, @@ -85,7 +87,7 @@ def installed(name, version=None, refresh=False, repo='', skip_verify=False): 'comment': 'Package {0} installed'.format(name)} -def latest(name, refresh=False, repo='', skip_verify=False): +def latest(name, refresh=False, repo='', skip_verify=False, **kwargs): ''' Verify that the named package is installed and the latest available package. If the package can be updated this state function will update @@ -126,14 +128,16 @@ def latest(name, refresh=False, repo='', skip_verify=False): ret['changes'] = __salt__['pkg.install'](name, True, repo=repo, - skip_verify=skip_verify) + skip_verify=skip_verify, + **kwargs) if os.path.isfile(rtag): os.remove(rtag) else: ret['changes'] = __salt__['pkg.install'](name, repo=repo, - skip_verify=skip_verify) + skip_verify=skip_verify, + **kwargs) if ret['changes']: ret['comment'] = 'Package {0} upgraded to latest'.format(name) From 9f0f8c8ebc88113691672c0c82266f5fa4c94822 Mon Sep 17 00:00:00 2001 From: Gordon McAllister <gordon.mcallister@gmail.com> Date: Tue, 28 Feb 2012 16:01:48 -0800 Subject: [PATCH 135/598] Fix a few typos and docstring/comment formatting --- salt/__init__.py | 2 +- salt/client.py | 48 +++++++++++++++++++------------------ salt/config.py | 9 +++---- salt/crypt.py | 39 ++++++++++++++++-------------- salt/exceptions.py | 12 +++++----- salt/loader.py | 14 +++++------ salt/minion.py | 53 +++++++++++++++++++++-------------------- salt/output.py | 11 ++++----- salt/payload.py | 7 +++--- salt/state.py | 59 +++++++++++++++++++++++----------------------- 10 files changed, 131 insertions(+), 123 deletions(-) diff --git a/salt/__init__.py b/salt/__init__.py index cacee983704a..515180276fc0 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -49,7 +49,7 @@ def verify_env(dirs): mode = os.stat(dir_) # TODO: Should this log if it can't set the permissions - # to very secure for these PKI cert directories? + # to very secure for these PKI cert directories? if not stat.S_IMODE(mode.st_mode) == 448: if os.access(dir_, os.W_OK): os.chmod(dir_, 448) diff --git a/salt/client.py b/salt/client.py index 335dc4b496be..4b842c6d36c1 100644 --- a/salt/client.py +++ b/salt/client.py @@ -11,17 +11,19 @@ # The components here are simple, and they need to be and stay simple, we # want a client to have 3 external concerns, and maybe a forth configurable # option. -# The concerns are +# The concerns are: # 1. Who executes the command? -# 2. what is the function being run? +# 2. What is the function being run? # 3. What arguments need to be passed to the function? # 4. How long do we wait for all of the replies? # # Next there are a number of tasks, first we need some kind of authentication -# This Client initially will be the master root client, which will run as the -# root user on the master server. +# This Client initially will be the master root client, which will run as +# the root user on the master server. +# # BUT we also want a client to be able to work over the network, so that # controllers can exist within disparate applications. +# # The problem is that this is a security nightmare, so I am going to start # small, and only start with the ability to execute salt commands locally. # This means that the primary client to build is, the LocalClient @@ -221,8 +223,8 @@ def cmd_full_return( def get_iter_returns(self, jid, minions, timeout=None): ''' - This method starts off a watcher looking at the return data for a - specified jid, it returns all of the information for the jid + This method starts off a watcher looking at the return data for + a specified jid, it returns all of the information for the jid ''' if timeout is None: timeout = self.opts['timeout'] @@ -273,8 +275,8 @@ def get_iter_returns(self, jid, minions, timeout=None): def get_returns(self, jid, minions, timeout=None): ''' - This method starts off a watcher looking at the return data for a - specified jid + This method starts off a watcher looking at the return data for + a specified jid ''' if timeout is None: timeout = self.opts['timeout'] @@ -320,8 +322,8 @@ def get_returns(self, jid, minions, timeout=None): def get_full_returns(self, jid, minions, timeout=None): ''' - This method starts off a watcher looking at the return data for a - specified jid, it returns all of the information for the jid + This method starts off a watcher looking at the return data for + a specified jid, it returns all of the information for the jid ''' if timeout is None: timeout = self.opts['timeout'] @@ -397,10 +399,10 @@ def find_cmd(self, cmd): def check_minions(self, expr, expr_form='glob'): ''' - Check the passed regex against the available minions' public - keys stored for authentication. This should return a set of ids - which match the regex, this will then be used to parse the - returns to make sure everyone has checked back in. + Check the passed regex against the available minions' public keys + stored for authentication. This should return a set of ids which + match the regex, this will then be used to parse the returns to + make sure everyone has checked back in. ''' return {'glob': self._check_glob_minions, 'pcre': self._check_pcre_minions, @@ -418,19 +420,19 @@ def pub(self, tgt, fun, arg=(), expr_form='glob', Arguments: tgt: The tgt is a regex or a glob used to match up the ids on - the minions. Salt works by always publishing every command to - all of the minions and then the minions determine if the - command is for them based on the tgt value. + the minions. Salt works by always publishing every command + to all of the minions and then the minions determine if + the command is for them based on the tgt value. fun: - The function name to be called on the remote host(s), this must - be a string in the format "<modulename>.<function name>" + The function name to be called on the remote host(s), this + must be a string in the format "<modulename>.<function name>" arg: - The arg option needs to be a tuple of arguments to pass to the - calling function, if left blank + The arg option needs to be a tuple of arguments to pass + to the calling function, if left blank Returns: jid: - A string, as returned by the publisher, which is the job id, - this will inform the client where to get the job results + A string, as returned by the publisher, which is the job + id, this will inform the client where to get the job results minions: A set, the targets that the tgt passed should match. ''' diff --git a/salt/config.py b/salt/config.py index 5c7c699df101..035f79af6999 100644 --- a/salt/config.py +++ b/salt/config.py @@ -163,8 +163,9 @@ def minion_config(path): opts['master_uri'] = 'tcp://' + opts['master_ip'] + ':'\ + str(opts['master_port']) - # Enabling open mode requires that the value be set to True, and nothing - # else! + # Enabling open mode requires that the value be set to True, and + # nothing else! + opts['open_mode'] = opts['open_mode'] is True # set up the extension_modules location from the cachedir @@ -225,8 +226,8 @@ def master_config(path): # Prepend root_dir to other paths prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', 'sock_dir']) - # Enabling open mode requires that the value be set to True, and nothing - # else! + # Enabling open mode requires that the value be set to True, and + # nothing else! opts['open_mode'] = opts['open_mode'] is True opts['auto_accept'] = opts['auto_accept'] is True opts['file_roots'] = _validate_file_roots(opts['file_roots']) diff --git a/salt/crypt.py b/salt/crypt.py index 7b2b9b1e7185..04a10f4e44c6 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -95,8 +95,8 @@ def __gen_token(self): class Auth(object): ''' - The Auth class provides the sequence for setting up communication with the - master server from a minion. + The Auth class provides the sequence for setting up communication with + the master server from a minion. ''' def __init__(self, opts): self.opts = opts @@ -124,9 +124,9 @@ def get_priv_key(self): def minion_sign_in_payload(self): ''' - Generates the payload used to authenticate with the master server. This - payload consists of the passed in id_ and the ssh public key to encrypt - the AES key sent back form the master. + Generates the payload used to authenticate with the master + server. This payload consists of the passed in id_ and the ssh + public key to encrypt the AES key sent back form the master. ''' payload = {} key = self.get_priv_key() @@ -142,8 +142,10 @@ def minion_sign_in_payload(self): def decrypt_aes(self, aes): ''' - This function is used to decrypt the aes seed phrase returned from the - master server, the seed phrase is decrypted with the ssh rsa host key. + This function is used to decrypt the aes seed phrase returned from + the master server, the seed phrase is decrypted with the ssh rsa + host key. + Pass in the encrypted aes key. Returns the decrypted aes seed key, a string ''' @@ -155,10 +157,11 @@ def verify_master(self, master_pub, token): ''' Takes the master pubkey and compares it to the saved master pubkey, the token is encrypted with the master private key and must be - decrypted successfully to verify that the master has been connected to. - The token must decrypt with the public key, and it must say: + decrypted successfully to verify that the master has been connected + to. The token must decrypt with the public key, and it must say: 'salty bacon' - returns a bool + + Returns a bool ''' tmp_pub = tempfile.mktemp() open(tmp_pub, 'w+').write(master_pub) @@ -304,8 +307,8 @@ def loads(self, data): class SAuth(Auth): ''' - Set up an object to maintain the standalone authentication session with - the salt master + Set up an object to maintain the standalone authentication session + with the salt master ''' def __init__(self, opts): super(SAuth, self).__init__(opts) @@ -314,20 +317,20 @@ def __init__(self, opts): def __authenticate(self): ''' Authenticate with the master, this method breaks the functional - paradigm, it will update the master information from a fresh sign in, - signing in can occur as often as needed to keep up with the revolving - master aes key. + paradigm, it will update the master information from a fresh sign + in, signing in can occur as often as needed to keep up with the + revolving master aes key. ''' creds = self.sign_in() if creds == 'retry': - log.error('Failed to authenticate with the master, verify that this'\ + log.error('Failed to authenticate with the master, verify this'\ + ' minion\'s public key has been accepted on the salt master') sys.exit(2) return Crypticle(self.opts, creds['aes']) def gen_token(self, clear_tok): ''' - Encrypt a string with the minion private key to verify identity with - the master. + Encrypt a string with the minion private key to verify identity + with the master. ''' return self.get_priv_key().private_encrypt(clear_tok, 5) diff --git a/salt/exceptions.py b/salt/exceptions.py index 341b90b2369b..79ebe28921df 100644 --- a/salt/exceptions.py +++ b/salt/exceptions.py @@ -28,8 +28,8 @@ class CommandNotFoundError(SaltException): class CommandExecutionError(SaltException): ''' - Used when a module runs a command which returns an error and - wants to show the user the output gracefully instead of dying + Used when a module runs a command which returns an error and wants + to show the user the output gracefully instead of dying ''' pass @@ -47,14 +47,14 @@ class MinionError(SaltException): class SaltInvocationError(SaltException): ''' - Used when the wrong number of arguments are sent to modules - or invalid arguments are specified on the command line + Used when the wrong number of arguments are sent to modules or invalid + arguments are specified on the command line ''' pass class PkgParseError(SaltException): ''' - Used when of the pkg modules cannot correctly parse the output from the CLI - tool (pacman, yum, apt, aptitude, etc) + Used when of the pkg modules cannot correctly parse the output from + the CLI tool (pacman, yum, apt, aptitude, etc) ''' pass diff --git a/salt/loader.py b/salt/loader.py index 4bfd3ef18741..fc57f9803fbb 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -173,9 +173,9 @@ def runner(opts): class Loader(object): ''' - Used to load in arbitrary modules from a directory, the Loader can also be - used to only load specific functions from a directory, or to call modules - in an arbitrary directory directly. + Used to load in arbitrary modules from a directory, the Loader can + also be used to only load specific functions from a directory, or to + call modules in an arbitrary directory directly. ''' def __init__(self, module_dirs, opts=dict(), tag='module'): self.module_dirs = module_dirs @@ -470,8 +470,8 @@ def filter_func(self, name, pack=None): def chop_mods(self): ''' - Chop off the module names so that the raw functions are exposed, used - to generate the grains + Chop off the module names so that the raw functions are exposed, + used to generate the grains ''' funcs = {} for key, fun in self.gen_functions().items(): @@ -481,8 +481,8 @@ def chop_mods(self): def gen_grains(self): ''' Read the grains directory and execute all of the public callable - members. Then verify that the returns are python dict's and return a - dict containing all of the returned values. + members. Then verify that the returns are python dict's and return + a dict containing all of the returned values. ''' grains = {} funcs = self.gen_functions() diff --git a/salt/minion.py b/salt/minion.py index 391fc505b72d..20b59d0df325 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -61,9 +61,9 @@ def get_proc_dir(cachedir): class SMinion(object): ''' Create an object that has loaded all of the minion module functions, - grains, modules, returners etc. - The SMinion allows developers to generate all of the salt minion functions - and present them with these functions for general use. + grains, modules, returners etc. The SMinion allows developers to + generate all of the salt minion functions and present them with these + functions for general use. ''' def __init__(self, opts): # Generate all of the minion side components @@ -83,8 +83,8 @@ def gen_modules(self): class Minion(object): ''' - This class instantiates a minion, runs connections for a minion, and loads - all of the functions into the minion + This class instantiates a minion, runs connections for a minion, + and loads all of the functions into the minion ''' def __init__(self, opts): ''' @@ -115,7 +115,8 @@ def __prep_mod_opts(self): def __load_modules(self): ''' - Return the functions and the returners loaded up from the loader module + Return the functions and the returners loaded up from the loader + module ''' self.opts['grains'] = salt.loader.grains(self.opts) functions = salt.loader.minion_mods(self.opts) @@ -177,8 +178,8 @@ def _handle_clear(self, load): def _handle_decoded_payload(self, data): ''' - Override this method if you wish to handle the decoded - data differently. + Override this method if you wish to handle the decoded data + differently. ''' if isinstance(data['fun'], basestring): if data['fun'] == 'sys.reload_modules': @@ -372,9 +373,9 @@ def _return_pub(self, ret, ret_cmd='_return'): def authenticate(self): ''' Authenticate with the master, this method breaks the functional - paradigm, it will update the master information from a fresh sign in, - signing in can occur as often as needed to keep up with the revolving - master aes key. + paradigm, it will update the master information from a fresh sign + in, signing in can occur as often as needed to keep up with the + revolving master aes key. ''' log.debug('Attempting to authenticate with the Salt Master') auth = salt.crypt.Auth(self.opts) @@ -447,8 +448,8 @@ def tune_in(self): class Syndic(salt.client.LocalClient, Minion): ''' - Make a Syndic minion, this minion will use the minion keys on the master to - authenticate with a higher level master. + Make a Syndic minion, this minion will use the minion keys on the + master to authenticate with a higher level master. ''' def __init__(self, opts): self._syndic = True @@ -479,8 +480,8 @@ def _handle_aes(self, load): def _handle_decoded_payload(self, data): ''' - Override this method if you wish to handle - the decoded data differently. + Override this method if you wish to handle the decoded data + differently. ''' if self.opts['multiprocessing']: multiprocessing.Process( @@ -650,13 +651,13 @@ def compound_match(self, tgt): # We didn't match a target, so append a boolean operator results.append(match) else: - # The match is not explicitely defined, evaluate it as a glob + # The match is not explicitly defined, evaluate it as a glob results.append(str(self.glob_match(match))) return eval(' '.join(results)) def nodegroup_match(self, tgt, nodegroups): ''' - This is a compatability matcher and is NOT called when using + This is a compatibility matcher and is NOT called when using nodegroups for remote execution, but is called when the nodegroups matcher is used in states ''' @@ -778,7 +779,7 @@ def get_dir(self, path, dest='', env='base'): ret = [] # Strip trailing slash path = string.rstrip(self._check_proto(path), '/') - # Break up the path into a list conaining the bottom-level directory + # Break up the path into a list containing the bottom-level directory # (the one being recursively copied) and the directories preceding it separated = string.rsplit(path,'/',1) if len(separated) != 2: @@ -850,15 +851,15 @@ def get_url(self, url, dest, makedirs=False, env='base'): def cache_file(self, path, env='base'): ''' - Pull a file down from the file server and store it in the minion file - cache + Pull a file down from the file server and store it in the minion + file cache ''' return self.get_url(path, '', True, env) def cache_files(self, paths, env='base'): ''' - Download a list of files stored on the master and put them - in the minion file cache + Download a list of files stored on the master and put them in the + minion file cache ''' ret = [] for path in paths: @@ -953,8 +954,8 @@ def is_cached(self, path, env='base'): def hash_file(self, path, env='base'): ''' - Return the hash of a file, to get the hash of a file on the - salt master file server prepend the path with salt://<file on server> + Return the hash of a file, to get the hash of a file on the salt + master file server prepend the path with salt://<file on server> otherwise, prepend the file with / for a local file. ''' try: @@ -991,8 +992,8 @@ def list_env(self, path, env='base'): def get_state(self, sls, env): ''' - Get a state file from the master and store it in the local minion cache - return the location of the file + Get a state file from the master and store it in the local minion + cache return the location of the file ''' if '.' in sls: sls = sls.replace('.', '/') diff --git a/salt/output.py b/salt/output.py index 45425b318b84..be49fa298e02 100644 --- a/salt/output.py +++ b/salt/output.py @@ -43,9 +43,9 @@ def __call__(self, data, **kwargs): class HighStateOutputter(Outputter): ''' - Not a command line option, the HighStateOutputter is only meant to be used - with the state.highstate function, or a function that returns highstate - return data + Not a command line option, the HighStateOutputter is only meant to + be used with the state.highstate function, or a function that returns + highstate return data ''' supports = 'highstate' @@ -127,9 +127,8 @@ def __call__(self, data, **kwargs): class TxtOutputter(Outputter): ''' - Plain text output. Primarily for returning output from - shell commands in the exact same way they would output - on the shell when ran directly. + Plain text output. Primarily for returning output from shell commands + in the exact same way they would output on the shell when ran directly. ''' supports = 'txt' diff --git a/salt/payload.py b/salt/payload.py index 3a5b19c9059c..d47ee630a58e 100644 --- a/salt/payload.py +++ b/salt/payload.py @@ -1,6 +1,7 @@ ''' Many aspects of the salt payload need to be managed, from the return of -encrypted keys to general payload dynamics and packaging, these happen in here +encrypted keys to general payload dynamics and packaging, these happen +in here ''' import cPickle as pickle @@ -10,8 +11,8 @@ def package(payload): ''' - This method for now just wraps msgpack.dumps, but it is here so that we can - make the serialization a custom option in the future with ease. + This method for now just wraps msgpack.dumps, but it is here so that + we can make the serialization a custom option in the future with ease. ''' return msgpack.dumps(payload) diff --git a/salt/state.py b/salt/state.py index c4e08f513d93..7607e46a68ac 100644 --- a/salt/state.py +++ b/salt/state.py @@ -111,8 +111,8 @@ def __init__(self, opts): def _mod_init(self, low): ''' - Check the module initilization function, if this is the first run of - a state package that has a mod_init function, then execute the + Check the module initialization function, if this is the first run + of a state package that has a mod_init function, then execute the mod_init function in the state module. ''' minit = '{0}.mod_init'.format(low['state']) @@ -155,9 +155,9 @@ def module_refresh(self, data): ''' Check to see if the modules for this state instance need to be updated, only update if the state is a file. If the function is - managed check to see if the file is a possible module type, eg a - python, pyx, or .so. Always refresh if the function is recuse, since - that can lay down anything. + managed check to see if the file is a possible module type, e.g. a + python, pyx, or .so. Always refresh if the function is recuse, + since that can lay down anything. ''' if data['state'] == 'file': if data['fun'] == 'managed': @@ -187,8 +187,8 @@ def module_refresh(self, data): def format_verbosity(self, returns): ''' - Check for the state_verbose option and strip out the result=True and - changes={} members of the state return list. + Check for the state_verbose option and strip out the result=True + and changes={} members of the state return list. ''' if self.opts['state_verbose']: return returns @@ -548,8 +548,8 @@ def template_shebang(self, template): def compile_template(self, template, env='', sls=''): ''' - Take the path to a template and return the high data structure derived - from the template. + Take the path to a template and return the high data structure + derived from the template. ''' if not isinstance(template, basestring): return {} @@ -559,8 +559,8 @@ def compile_template(self, template, env='', sls=''): def compile_template_str(self, template): ''' - Take the path to a template and return the high data structure derived - from the template. + Take the path to a template and return the high data structure + derived from the template. ''' fn_ = tempfile.mkstemp()[1] open(fn_, 'w+').write(template) @@ -570,8 +570,8 @@ def compile_template_str(self, template): def call(self, data): ''' - Call a state directly with the low data structure, verify data before - processing. + Call a state directly with the low data structure, verify data + before processing. ''' log.info( 'Executing state {0[state]}.{0[fun]} for {0[name]}'.format( @@ -621,7 +621,8 @@ def check_failhard(self, low, running): def check_requisite(self, low, running, chunks): ''' - Look into the running data to check the status of all requisite states + Look into the running data to check the status of all requisite + states ''' present = False if 'watch' in low: @@ -671,8 +672,8 @@ def check_requisite(self, low, running, chunks): def call_chunk(self, low, running, chunks): ''' - Check if a chunk has any requires, execute the requires and then the - chunk + Check if a chunk has any requires, execute the requires and then + the chunk ''' self._mod_init(low) tag = _gen_tag(low) @@ -784,9 +785,9 @@ def call_template_str(self, template): class HighState(object): ''' - Generate and execute the salt "High State". The High State is the compound - state derived from a group of template files stored on the salt master or - in the local cache. + Generate and execute the salt "High State". The High State is the + compound state derived from a group of template files stored on the + salt master or in the local cache. ''' def __init__(self, opts): self.client = salt.minion.FileClient(opts) @@ -796,9 +797,9 @@ def __init__(self, opts): def __gen_opts(self, opts): ''' - The options used by the High State object are derived from options on - the minion and the master, or just the minion if the high state call is - entirely local. + The options used by the High State object are derived from options + on the minion and the master, or just the minion if the high state + call is entirely local. ''' # If the state is intended to be applied locally, then the local opts # should have all of the needed data, otherwise overwrite the local @@ -930,8 +931,8 @@ def get_top(self): def top_matches(self, top): ''' - Search through the top high data for matches and return the states that - this minion needs to execute. + Search through the top high data for matches and return the states + that this minion needs to execute. Returns: {'env': ['state1', 'state2', ...]} @@ -993,7 +994,7 @@ def render_state(self, sls, env, mods): errors = [] fn_ = self.client.get_state(sls, env) if not fn_: - errors.append(('Specifed SLS {0} in environment {1} is not' + errors.append(('Specified SLS {0} in environment {1} is not' ' available on the salt master').format(sls, env)) state = None try: @@ -1056,8 +1057,8 @@ def render_state(self, sls, env, mods): def render_highstate(self, matches): ''' - Gather the state files and render them into a single unified salt high - data structure. + Gather the state files and render them into a single unified salt + high data structure. ''' highstate = {} errors = [] @@ -1121,8 +1122,8 @@ def compile_highstate(self): def compile_low_chunks(self): ''' - Compile the highstate but don't run it, return the low chunks to see - exactly what the highstate will execute + Compile the highstate but don't run it, return the low chunks to + see exactly what the highstate will execute ''' err = [] top = self.get_top() From 683e74ea7ca215fadab83ccdd05491627d17b196 Mon Sep 17 00:00:00 2001 From: Robert James Hernandez <herna175@mail.chapman.edu> Date: Tue, 28 Feb 2012 16:18:30 -0800 Subject: [PATCH 136/598] Corrected CLI example thanks to SEJeff! --- salt/modules/freebsdpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/freebsdpkg.py b/salt/modules/freebsdpkg.py index 6da1c5c80b22..ef6b264c6f14 100644 --- a/salt/modules/freebsdpkg.py +++ b/salt/modules/freebsdpkg.py @@ -191,7 +191,7 @@ def rehash(): CLI Example:: - salt '*' cmd.run 'rehash' + salt '*' pkg.rehash ''' shell = __salt__['cmd.run']('echo $SHELL').split('/') if shell[len(shell)-1] in ["csh","tcsh"]: From dcebaac7f766ddb358232611b82e5e550dca0ad8 Mon Sep 17 00:00:00 2001 From: Grier Johnson <grierj@gmail.com> Date: Tue, 28 Feb 2012 16:45:01 -0800 Subject: [PATCH 137/598] Pass the user in the load Forgot to add the actual user in the payload so the minion can read it out for logging. --- salt/master.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/master.py b/salt/master.py index a7f1e9ecfe5c..789ab3cdb9ad 100644 --- a/salt/master.py +++ b/salt/master.py @@ -857,6 +857,7 @@ def publish(self, clear_load): if 'user' in clear_load: log.info(('User {0[user]} Published command {0[fun]} with jid' ' {0[jid]}').format(clear_load)) + load['user'] = clear_load['user'] else: log.info(('Published command {0[fun]} with jid' ' {0[jid]}').format(clear_load)) From ff62c63ad4be15e4a14f2db2bee4dc9365844bde Mon Sep 17 00:00:00 2001 From: Christer Edwards <christer.edwards@gmail.com> Date: Tue, 28 Feb 2012 22:26:46 -0700 Subject: [PATCH 138/598] more formatting and syntax cleanup in release doc --- doc/topics/releases/0.9.7.rst | 58 +++++++++++++++++------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/topics/releases/0.9.7.rst b/doc/topics/releases/0.9.7.rst index b0f27ac2a6ae..87429d163722 100644 --- a/doc/topics/releases/0.9.7.rst +++ b/doc/topics/releases/0.9.7.rst @@ -8,7 +8,7 @@ under the hood, as well as some features that make working with Salt much better. A few highlights include the new Job system, refinements to the requisite -system in states, the mod_init interface for states, external node +system in states, the ``mod_init`` interface for states, external node classification, search path to managed files in the file state, and refinements and additions to dynamic module loading. @@ -24,12 +24,12 @@ Salt Jobs Interface The new jobs interface makes the management of running executions much cleaner and more transparent. Building on the existing execution framework the jobs -system allows clear introspection into the active running state of the, +system allows clear introspection into the active running state of the running Salt interface. The Jobs interface is centered in the new minion side proc system. The -minions now store a msgpack serialized file under /var/cache/salt/proc. These -files keep track of the active state of processes on the minion. +minions now store msgpack serialized files under ``/var/cache/salt/proc``. +These files keep track of the active state of processes on the minion. Functions in the saltutil Module ```````````````````````````````` @@ -37,18 +37,18 @@ Functions in the saltutil Module A number of functions have been added to the saltutil module to manage and view the jobs: -1. running - Returns the data of all running jobs that are found in the proc directory. -2. find_job - Returns specific data about a certain job based on job id. -3. signal_job - Allows for a given jid to be sent a signal. -4. term_job - Sends a termination signal (SIGTERM, 15) to the process controlling the - specified job. -5. kill_job - Sends a kill signal (SIGKILL, 9) to the process controlling the - specified job. +``running`` - Returns the data of all running jobs that are found in the proc +directory. + +``find_job`` - Returns specific data about a certain job based on job id. + +``signal_job`` - Allows for a given jid to be sent a signal. + +``term_job`` - Sends a termination signal (``SIGTERM, 15``) to the process +controlling the specified job. + +``kill_job`` Sends a kill signal (``SIGKILL, 9``) to the process controlling the +specified job. The jobs Runner --------------- @@ -61,7 +61,7 @@ The jobs runner contains a number of functions... active `````` -The active function runs saltutil.running on all minions and formats the +The active function runs ``saltutil.running`` on all minions and formats the return data about all running jobs in a much more usable and compact format. The active function will also compare jobs that have returned and jobs that are still running, making it easier to see what systems have completed a job @@ -73,21 +73,22 @@ lookup_jid When jobs are executed the return data is sent back to the master and cached. By default is is cached for 24 hours, but this can be configured via the ``keep_jobs`` option in the master configuration. -Using the lookup_jid runner will display the same return data that the initial -job invocation with the salt command would display. + +Using the ``lookup_jid`` runner will display the same return data that the +initial job invocation with the salt command would display. list_jobs ````````` -Before finding a historic job, it may be required to find the job id. list_jobs -will parse the cached execution data and display all of the job data for jobs -that have already, or partially returned. +Before finding a historic job, it may be required to find the job id. +``list_jobs`` will parse the cached execution data and display all of the job +data for jobs that have already, or partially returned. External Node Classification ---------------------------- -Salt can now use external node classifiers like Cobbler's +Salt can now use external node classifiers like Cobbler's ``cobbler-ext-nodes``. Salt uses specific data from the external node classifier. In particular the @@ -107,14 +108,14 @@ State Mod Init System An issue arose with the pkg state. Every time a package was run Salt would need to refresh the package database. This made systems with slower package metadata refresh speeds much slower to work with. To alleviate this issue the -mod_init interface has been added to salt states. +``mod_init`` interface has been added to salt states. -The mod_init interface is a function that can be added to a state file. +The ``mod_init`` interface is a function that can be added to a state file. This function is called with the first state called. In the case of the pkg -state, the mod_init function sets up a tag which makes the package database +state, the ``mod_init`` function sets up a tag which makes the package database only refresh on the first attempt to install a package. -In a nutshell, the mod_init interface allows a state to run any command that +In a nutshell, the ``mod_init`` interface allows a state to run any command that only needs to be run once, or can be used to set up an environment for working with the state. @@ -153,7 +154,7 @@ more quickly. Initial Unit Testing Framework ------------------------------ -Because of the module system, and the need to test real scenarios the, +Because of the module system, and the need to test real scenarios, the development of a viable unit testing system has been difficult, but unit testing has finally arrived. Only a small amount of unit testing coverage has been developed, much more coverage will be in place soon. @@ -176,4 +177,3 @@ be used to match nodes for states. The nodegroups support has been expanded and the nodegroups defined in the master configuration can now be used to match minions in the top file. - From ab9c5fefc9ab5a8d6ab894a76ff55a0d52c8568c Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Tue, 28 Feb 2012 23:05:37 -0700 Subject: [PATCH 139/598] Add win_network.py Beginning of win_network.py. Three methods are finished. There are several more to convert to work with Windows --- salt/modules/network.py | 10 ++ salt/modules/win_network.py | 302 ++++++++++++++++++++++++++++++++++++ 2 files changed, 312 insertions(+) create mode 100644 salt/modules/win_network.py diff --git a/salt/modules/network.py b/salt/modules/network.py index 15f84c067c0b..f2c9f6aa2ece 100644 --- a/salt/modules/network.py +++ b/salt/modules/network.py @@ -13,6 +13,16 @@ 'netstat': 'txt', } +def __virtual__(): + ''' + Only work on posix-like systems + ''' + + # Disable on Windows, a specific file module exists: + if __grains__['os'] == 'Windows': + return False + return 'network' + def _sanitize_host(host): ''' diff --git a/salt/modules/win_network.py b/salt/modules/win_network.py new file mode 100644 index 000000000000..8c6c687945a2 --- /dev/null +++ b/salt/modules/win_network.py @@ -0,0 +1,302 @@ +''' +Module for gathering and managing network information +''' + +from string import ascii_letters, digits +import socket +import re +import salt.utils + +__outputter__ = { + 'dig': 'txt', + 'ping': 'txt', + 'netstat': 'txt', +} + +def __virtual__(): + ''' + Only works on Windows systems + ''' + if __grains__['os'] == 'Windows': + return 'network' + return False + + + +def _sanitize_host(host): + ''' + Sanitize host string. + ''' + return "".join([ + c for c in host[0:255] if c in (ascii_letters + digits + '.') + ]) + + +def ping(host): + ''' + Performs a ping to a host + + CLI Example:: + + salt '*' network.ping archlinux.org + ''' + cmd = 'ping -n 4 %s' % _sanitize_host(host) + return __salt__['cmd.run'](cmd) + + +def netstat(): + ''' + Return information on open ports and states + + CLI Example:: + + salt '*' network.netstat + ''' + ret = [] + cmd = 'netstat -na' + lines = __salt__['cmd.run'](cmd).split('\n') + for line in lines: + comps = line.split() + if line.startswith(' TCP'): + ret.append({ + 'local-address': comps[1], + 'proto': comps[0], + 'remote-address': comps[2], + 'state': comps[3]}) + if line.startswith(' UDP'): + ret.append({ + 'local-address': comps[1], + 'proto': comps[0], + 'remote-address': comps[2], + 'state': None}) + return ret + + +def traceroute(host): + ''' + Performs a traceroute to a 3rd party host + + CLI Example:: + + salt '*' network.traceroute archlinux.org + ''' + ret = [] + cmd = 'tracert %s' % _sanitize_host(host) + lines = __salt__['cmd.run'](cmd).split('\n') + for line in lines: + if not ' ' in line: + continue + if line.startswith('Trac'): + continue + if line.startswith('over'): + continue + comps = line.split() + complength = len(comps) + if complength == 9: + result = { + 'count': comps[0], + 'hostname': comps[7], + 'ip': comps[8], + 'ms1': comps[1], + 'ms2': comps[3], + 'ms3': comps[5]} + ret.append(result) + elif complength == 8: + result = { + 'count': comps[0], + 'hostname': None, + 'ip': comps[7], + 'ms1': comps[1], + 'ms2': comps[3], + 'ms3': comps[5]} + ret.append(result) + else: + result = { + 'count': comps[0], + 'hostname': None, + 'ip': None, + 'ms1': None, + 'ms2': None, + 'ms3': None} + ret.append(result) + return ret + + +def dig(host): + ''' + Performs a DNS lookup with dig + + CLI Example:: + + salt '*' network.dig archlinux.org + ''' + cmd = 'dig %s' % _sanitize_host(host) + return __salt__['cmd.run'](cmd) + + +def isportopen(host, port): + ''' + Return status of a port + + CLI Example:: + + salt '*' network.isportopen 127.0.0.1 22 + ''' + + if not (1 <= int(port) <= 65535): + return False + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + out = sock.connect_ex((_sanitize_host(host), int(port))) + + return out + + +def _cidr_to_ipv4_netmask(cidr_bits): + ''' + Returns an IPv4 netmask + ''' + netmask = '' + for n in range(4): + if n: + netmask += '.' + if cidr_bits >= 8: + netmask += '255' + cidr_bits -= 8 + else: + netmask += '%d' % (256-(2**(8-cidr_bits))) + cidr_bits = 0 + return netmask + + +def interfaces(): + ''' + Returns a dictionary of interfaces with various information about each + (up/down state, ip address, netmask, and hwaddr) + + CLI Example:: + + salt '*' network.interfaces + ''' + ret = {} + + out = __salt__['cmd.run']('ip addr show') + groups = re.compile('\r?\n\d').split(out) + + for group in groups: + iface = None + data = {} + for line in group.split('\n'): + if not ' ' in line: + continue + m = re.match('^\d*:\s+(\w+)(?:@)?(\w+)?:\s+<(.+)>', line) + if m: + iface,parent,attrs = m.groups() + if 'UP' in attrs.split(','): + data['up'] = True + if parent: + data['parent'] = parent + else: + cols = line.split() + if len(cols) >= 2: + type,value = tuple(cols[0:2]) + if type in ('inet', 'inet6'): + def parse_network(): + """ + Return a tuple of ip, netmask, broadcast + based on the current set of cols + """ + brd = None + ip,cidr = tuple(value.split('/')) + if type == 'inet': + mask = _cidr_to_ipv4_netmask(int(cidr)) + if 'brd' in cols: + brd = cols[cols.index('brd')+1] + elif type == 'inet6': + mask = cidr + return (ip, mask, brd) + + if 'secondary' not in cols: + ipaddr, netmask, broadcast = parse_network() + if type == 'inet': + data['ipaddr'] = ipaddr + data['netmask'] = netmask + data['broadcast'] = broadcast + elif type == 'inet6': + data['ipaddr6'] = ipaddr + data['netmask6'] = netmask + else: + if 'secondary' not in data: + data['secondary'] = [] + ip, mask, brd = parse_network() + data['secondary'].append({ + 'type': type, + 'ipaddr': ip, + 'netmask': mask, + 'broadcast': brd + }) + del ip, mask, brd + elif type.startswith('link'): + data['hwaddr'] = value + if iface: + ret[iface] = data + del iface, data + return ret + + +def up(interface): + ''' + Returns True if interface is up, otherwise returns False + + CLI Example:: + + salt '*' network.up eth0 + ''' + data = interfaces().get(interface) + if data: + return data['up'] + else: + return None + +def ipaddr(interface): + ''' + Returns the IP address for a given interface + + CLI Example:: + + salt '*' network.ipaddr eth0 + ''' + data = interfaces().get(interface) + if data: + return data['ipaddr'] + else: + return None + +def netmask(interface): + ''' + Returns the netmask for a given interface + + CLI Example:: + + salt '*' network.netmask eth0 + ''' + data = interfaces().get(interface) + if data: + return data['netmask'] + else: + return None + +def hwaddr(interface): + ''' + Returns the hwaddr for a given interface + + CLI Example:: + + salt '*' network.hwaddr eth0 + ''' + data = interfaces().get(interface) + if data: + return data['hwaddr'] + else: + return None + From ef593130430f5e75921e6ad3e2d6c3e5089f70a6 Mon Sep 17 00:00:00 2001 From: Erik Johnson <palehose@gmail.com> Date: Wed, 29 Feb 2012 00:10:35 -0600 Subject: [PATCH 140/598] add user comment fields and password hash enforcement control to file state --- salt/modules/useradd.py | 143 ++++++++++++++++++++++++++++++++++++++-- salt/states/user.py | 68 +++++++++++++++++-- 2 files changed, 202 insertions(+), 9 deletions(-) diff --git a/salt/modules/useradd.py b/salt/modules/useradd.py index e00615dd1582..23206195507d 100644 --- a/salt/modules/useradd.py +++ b/salt/modules/useradd.py @@ -18,7 +18,12 @@ def add(name, gid=None, groups=None, home=True, - shell=None): + shell=None, + fullname=None, + roomnumber=None, + workphone=None, + homephone=None, + other=None): ''' Add a user to the minion @@ -43,9 +48,29 @@ def add(name, else: cmd += '-m -d {0} '.format(home) cmd += name - ret = __salt__['cmd.run_all'](cmd) - - return not ret['retcode'] + ret = __salt__['cmd.retcode'](cmd) + if ret != 0: + return False + else: + # At this point, the user was successfully created, so return true + # regardless of the outcome of the below functions. If there is a + # problem wth changing any of the user's info below, it will be raised + # in a future highstate call. If anyone has a better idea on how to do + # this, feel free to change it, but I didn't think it was a good idea + # to return False when the user was successfully created since A) the + # user does exist, and B) running useradd again would result in a + # nonzero exit status and be interpreted as a False result. + if fullname: + chfullname(name,fullname) + if roomnumber: + chroomnumber(name,roomnumber) + if workphone: + chworkphone(name,workphone) + if homephone: + chhomephone(name,homephone) + if other: + chother(name,other) + return True def delete(name, remove=False, force=False): @@ -184,6 +209,101 @@ def chgroups(name, groups, append=False): return len(ugrps - agrps) == 0 +def chfullname(name, fullname): + ''' + Change the users Full Name + + CLI Example:: + + salt '*' user.chfullname foo "Foo Bar" + ''' + pre_info = info(name) + if fullname == pre_info['fullname']: + return True + cmd = 'chfn -f "{0}" {1}'.format(fullname,name) + __salt__['cmd.run'](cmd) + post_info = info(name) + if post_info['fullname'] != pre_info['fullname']: + return post_info['fullname'] == fullname + return False + + +def chroomnumber(name, roomnumber): + ''' + Change the user's Room Number + + CLI Example:: + + salt '*' user.chroomnumber foo 123 + ''' + pre_info = info(name) + if roomnumber == pre_info['roomnumber']: + return True + cmd = 'chfn -r "{0}" {1}'.format(roomnumber,name) + __salt__['cmd.run'](cmd) + post_info = info(name) + if post_info['roomnumber'] != pre_info['roomnumber']: + return post_info['roomnumber'] == roomnumber + return False + + +def chworkphone(name, workphone): + ''' + Change the user's Work Phone + + CLI Example:: + + salt '*' user.chworkphone foo "7735550123" + ''' + pre_info = info(name) + if workphone == pre_info['workphone']: + return True + cmd = 'chfn -w "{0}" {1}'.format(workphone,name) + __salt__['cmd.run'](cmd) + post_info = info(name) + if post_info['workphone'] != pre_info['workphone']: + return post_info['workphone'] == workphone + return False + + +def chhomephone(name, homephone): + ''' + Change the user's Home Phone + + CLI Example:: + + salt '*' user.chhomephone foo "7735551234" + ''' + pre_info = info(name) + if homephone == pre_info['homephone']: + return True + cmd = 'chfn -h "{0}" {1}'.format(homephone,name) + __salt__['cmd.run'](cmd) + post_info = info(name) + if post_info['homephone'] != pre_info['homephone']: + return post_info['homephone'] == homephone + return False + + +def chother(name, other): + ''' + Change the user's "Other" GECOS field + + CLI Example:: + + salt '*' user.chother foo "fax=7735555678" + ''' + pre_info = info(name) + if other == pre_info['other']: + return True + cmd = 'chfn -o "{0}" {1}'.format(other,name) + __salt__['cmd.run'](cmd) + post_info = info(name) + if post_info['other'] != pre_info['other']: + return post_info['other'] == other + return False + + def info(name): ''' Return user information @@ -202,6 +322,16 @@ def info(name): ret['passwd'] = data.pw_passwd ret['shell'] = data.pw_shell ret['uid'] = data.pw_uid + # Put GECOS info into a list + gecos_field = data.pw_gecos.split(',', 4) + # Assign empty strings for any unspecified GECOS fields + while len(gecos_field) < 5: + gecos_field.append('') + ret['fullname'] = gecos_field[0] + ret['roomnumber'] = gecos_field[1] + ret['workphone'] = gecos_field[2] + ret['homephone'] = gecos_field[3] + ret['other'] = gecos_field[4] except KeyError: ret['gid'] = '' ret['groups'] = '' @@ -210,6 +340,11 @@ def info(name): ret['passwd'] = '' ret['shell'] = '' ret['uid'] = '' + ret['fullname'] = '' + ret['roomnumber'] = '' + ret['workphone'] = '' + ret['homephone'] = '' + ret['other'] = '' return ret diff --git a/salt/states/user.py b/salt/states/user.py index 4e8a2640a7cd..46c13aab40d6 100644 --- a/salt/states/user.py +++ b/salt/states/user.py @@ -9,6 +9,7 @@ fred: user: - present + - fullname: Fred Jones - shell: /bin/zsh - home: /home/fred - uid: 4000 @@ -27,7 +28,13 @@ def present( groups=None, home=False, password=None, - shell=None + enforce_password=True, + shell=None, + fullname=None, + roomnumber=None, + workphone=None, + homephone=None, + other=None, ): ''' Ensure that the named user is present with the specified properties @@ -51,8 +58,36 @@ def present( password A password hash to set for the user + enforce_password + Set to False to keep the password from being changed if it has already + been set and the password hash differs from what is specified in the + "password" field. This option will be ignored if "password" is not + specified. + shell The login shell, defaults to the system default shell + + + User comment field (GECOS) support (currently Linux-only): + + The below values should be specified as strings to avoid ambiguities when + the values are loaded. (Especially the phone and room number fields which + are likely to contain numeric data) + + fullname + The user's full name. + + roomnumber + The user's room number + + workphone + The user's work phone number + + homephone + The user's home phone number + + other + The user's "other" GECOS field ''' ret = {'name': name, 'changes': {}, @@ -89,9 +124,31 @@ def present( __salt__['user.chshell'](name, shell) if password: if __grains__['os'] != 'FreeBSD': - if lshad['pwd'] != password: - # Set the new password - __salt__['shadow.set_password'](name, password) + if lshad['pwd'] == '!' or \ + lshad['pwd'] != '!' and enforce_password: + if lshad['pwd'] != password: + # Set the new password + __salt__['shadow.set_password'](name, password) + if fullname: + if lusr['fullname'] != fullname: + # Fix the fullname + __salt__['user.chfullname'](name, fullname) + if roomnumber: + if lusr['roomnumber'] != roomnumber: + # Fix the roomnumber + __salt__['user.chroomnumber'](name, roomnumber) + if workphone: + if lusr['workphone'] != workphone: + # Fix the workphone + __salt__['user.chworkphone'](name, workphone) + if homephone: + if lusr['homephone'] != homephone: + # Fix the homephone + __salt__['user.chhomephone'](name, homephone) + if other: + if lusr['other'] != other: + # Fix the other + __salt__['user.chother'](name, other) post = __salt__['user.info'](name) spost = {} if __grains__['os'] != 'FreeBSD': @@ -110,7 +167,8 @@ def present( return ret # The user is not present, make it! - if __salt__['user.add'](name, uid, gid, groups, home, shell): + if __salt__['user.add'](name, uid, gid, groups, home, shell, fullname, + roomnumber, workphone, homephone, other): ret['comment'] = 'New user {0} created'.format(name) ret['changes'] = __salt__['user.info'](name) if password: From ec79cffee08815337c17aefc895458a07df5e9d8 Mon Sep 17 00:00:00 2001 From: Christer Edwards <christer.edwards@gmail.com> Date: Wed, 29 Feb 2012 00:28:02 -0700 Subject: [PATCH 141/598] fixed link to freebsd installation tutorial --- doc/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.rst b/doc/index.rst index b3c0fffa7468..aea492541728 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -60,7 +60,7 @@ truly make it work for you. .. sidebar:: More tutorials! * :doc:`Bootstraping Salt on EC2 <topics/tutorials/bootstrap_ec2>` - * :doc:`Installing Salt on FreeBSD <topics/tutorials/freebsd>` + * :doc:`Installing Salt on FreeBSD <topics/installation/freebsd>` .. contents:: The components of Salt :local: From f2902790e38b85e3c34ce5ad41a34630df9b5684 Mon Sep 17 00:00:00 2001 From: Christer Edwards <christer.edwards@gmail.com> Date: Wed, 29 Feb 2012 13:39:07 -0700 Subject: [PATCH 142/598] initial attempt at a firewall doc for salt master --- doc/topics/tutorials/firewall.rst | 59 +++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 doc/topics/tutorials/firewall.rst diff --git a/doc/topics/tutorials/firewall.rst b/doc/topics/tutorials/firewall.rst new file mode 100644 index 000000000000..e19126c937c5 --- /dev/null +++ b/doc/topics/tutorials/firewall.rst @@ -0,0 +1,59 @@ +==================== +Opening the Firewall +==================== + +The Salt master communicates with the minions using an AES-encrypted ZeroMQ +connection. These communications are done over ports 4505 and 4506, which need +to be accessible on the master only. This document outlines suggested firewall +rules for allowing these incoming connections to the master. + +.. note:: + + **No firewall configuration needs to be done on Salt minions. These changes + refer to the master only.** + +iptables +======== + +Different Linux distributions store their iptables rules in different places, +which makes it difficult to standardize firewall documentation. I've included +some of the more common locations, but your mileage may vary. + +**Fedora / Red Hat / CentOS** :: + + /etc/sysconfig/iptables + +**Arch Linux** :: + + /etc/iptables/iptables.ruls + +**Debian/Ubuntu** :: + + ??? + +Once you've found your firewall rules, you'll need to add the two lines below +to allow traffic on ``tcp/4505`` and ``tcp/4506``: + +.. code-block:: diff + + + -A INPUT -m state --state new -m tcp -p tcp --dport 4505 -j ACCEPT + + -A INPUT -m state --state new -m tcp -p tcp --dport 4506 -j ACCEPT + +pf.conf +======= + +The BSD-family of operating systems uses packet filter (pf). The following +example describes the additions to ``pf.conf`` needed to access the Salt +master. + +.. code-block:: diff + + + pass in on $int_if proto tcp from any to $int_if port 4505 + + pass in on $int_if proto tcp from any to $int_if port 4506 + +Once you've made these additions to your ``pf.conf`` you'll need to reload the +new rules with the new additions. This can be done using the ``pfctl`` command. + +.. code-block:: bash + + pfctl -vf /etc/pf.conf From e78a0fc5b398ecae4c6cead3f9ce6ffa23d10aa6 Mon Sep 17 00:00:00 2001 From: Christer Edwards <christer.edwards@gmail.com> Date: Wed, 29 Feb 2012 13:41:38 -0700 Subject: [PATCH 143/598] standardized install docs for code-blocks --- doc/topics/installation/arch.rst | 82 +++++++++++++++++++---------- doc/topics/installation/debian.rst | 8 ++- doc/topics/installation/fedora.rst | 66 ++++++++++++++++------- doc/topics/installation/freebsd.rst | 39 +++++++++----- doc/topics/installation/gentoo.rst | 20 ++++--- 5 files changed, 147 insertions(+), 68 deletions(-) diff --git a/doc/topics/installation/arch.rst b/doc/topics/installation/arch.rst index 24c8a884bbec..3d6d30f979ce 100644 --- a/doc/topics/installation/arch.rst +++ b/doc/topics/installation/arch.rst @@ -18,7 +18,9 @@ currently stable and -git packages available. Stable Release -------------- -To install Salt stable releases from the Arch Linux AUR, use the commands:: +To install Salt stable releases from the Arch Linux AUR, use the commands: + +.. code-block:: bash wget https://aur.archlinux.org/packages/sa/salt/salt.tar.gz tar xf salt.tar.gz @@ -39,7 +41,7 @@ currently relies on the following packages only available via the AUR: The command to install salt using the yaourt tool is: - .. code-block:: none + .. code-block:: bash yaourt salt @@ -49,7 +51,9 @@ Tracking develop ---------------- To install the bleeding edge version of Salt (**may include bugs!**), you can use -the -git package. Installing the -git package can be done using the commands:: +the -git package. Installing the -git package can be done using the commands: + +.. code-block:: bash wget https://aur.archlinux.org/packages/sa/salt-git/salt-git.tar.gz tar xf salt-git.tar.gz @@ -70,7 +74,7 @@ currently relies on the following packages only available via the AUR: The command to install salt using the yaourt tool is: - .. code-block:: none + .. code-block:: bash yaourt salt-git @@ -82,9 +86,12 @@ Configuration In the sections below I'll outline configuration options for both the Salt Master and Salt Minions. -The Salt package installs two template configuration files, /etc/salt/master.template and -/etc/salt/minion.template. You'll need to copy these .template files into place and -make a few edits. First, copy them into place as seen here:: +The Salt package installs two template configuration files, +``/etc/salt/master.template`` and ``/etc/salt/minion.template``. You'll need +to copy these .template files into place and make a few edits. First, copy +them into place as seen here: + +.. code-block:: bash cp /etc/salt/master.template /etc/salt/master cp /etc/salt/minion.template /etc/salt/minion @@ -107,22 +114,30 @@ configuration paths. By default the Salt master listens on ports 4505 and 4506 on all interfaces (0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the -"interface" directive as seen here:: +"interface" directive as seen here: + +.. code-block:: diff - #interface: 0.0.0.0 + interface: 10.0.0.1 **rc.conf** -Last but not least you'll need to activate the Salt Master in your rc.conf -file. Using your favorite editor, open /etc/rc.conf and add the salt-master:: +You'll need to activate the Salt Master in your rc.conf file. Using your +favorite editor, open ``/etc/rc.conf`` and add the salt-master. + +.. code-block:: diff -DAEMONS=(syslog-ng network crond) +DAEMONS=(syslog-ng network crond @salt-master) +**Start the Master** + Once you've completed all of these steps you're ready to start your Salt Master. You should be able to start your Salt Master now using the command -seen here:: +seen here: + +.. code-block:: bash rc.d start salt-master @@ -145,7 +160,9 @@ this you likely can do without any minion configuration at all. If you are not able to update DNS, you'll simply need to update one entry in the configuration file. Using your favorite editor, open the minion -configuration file and update the "master" entry as seen here:: +configuration file and update the "master" entry as seen here. + +.. code-block:: diff - #master: salt + master: 10.0.0.1 @@ -157,15 +174,20 @@ configuration options are covered in another chapter. **rc.conf** Before you're able to start the Salt Minion you'll need to update your rc.conf -file. Using your favorite editor open /etc/rc.conf or /etc/rc.conf.local and -add this line:: +file. Using your favorite editor open ``/etc/rc.conf`` and add this line: + +.. code-block:: diff -DAEMONS=(syslog-ng network crond) +DAEMONS=(syslog-ng network crond @salt-minion) +**Start the Minion** + Once you've completed all of these steps you're ready to start your Salt Minion. You should be able to start your Salt Minion now using the command -seen here:: +seen here: + +.. code-block:: bash rc.d start salt-minion @@ -192,28 +214,32 @@ only done through trusted, accepted keys. Before you'll be able to do any remote execution or configuration management you'll need to accept any pending keys on the Master. Run the ``salt-key`` command to -list the keys known to the Salt Master:: +list the keys known to the Salt Master. + +.. code-block:: bash [root@master ~]# salt-key -L Unaccepted Keys: - avon - bodie - bubbles - marlo + alpha + bravo + charlie + delta Accepted Keys: This example shows that the Salt Master is aware of four Minions, but none of the keys have been accepted. To accept the keys and allow the Minions to be -controlled by the Master, again use the ``salt-key`` command:: +controlled by the Master, again use the ``salt-key`` command: + +.. code-block:: bash [root@master ~]# salt-key -A [root@master ~]# salt-key -L Unaccepted Keys: Accepted Keys: - avon - bodie - bubbles - marlo + alpha + bravo + charlie + delta The ``salt-key`` command allows for signing keys individually or in bulk. The example above, using ``-A`` bulk-accepts all pending keys. To accept keys @@ -228,10 +254,12 @@ Whether you have a few or a few-dozen, Salt can help you manage them easily! For final verification, send a test function from your Salt Master to your minions. If all of your minions are properly communicating with your Master, you should "True" responses from each of them. See the example below to send -the ``test.ping`` remote command:: +the ``test.ping`` remote command: + +.. code-block:: bash [root@master ~]# salt '*' test.ping - {'avon': True} + {'alpha': True} Where Do I Go From Here ======================== diff --git a/doc/topics/installation/debian.rst b/doc/topics/installation/debian.rst index 1c693d7bf01e..80ff4275933a 100644 --- a/doc/topics/installation/debian.rst +++ b/doc/topics/installation/debian.rst @@ -6,7 +6,9 @@ Ubuntu ------ We are working to get Salt into apt. In the meantime we have a PPA available -for Lucid:: +for Lucid: + +.. code-block:: bash aptitude -y install python-software-properties add-apt-repository ppa:saltstack/salt @@ -29,7 +31,9 @@ accepted you can install Salt by downloading the latest ``.deb`` in the There is a `python-zmq`__ package available in Debian \"wheezy (testing)\". If you don't have that repo enabled the best way to install Salt and pyzmq - is by using ``pip`` (or ``easy_install``):: + is by using ``pip`` (or ``easy_install``): + + .. code-block:: bash pip install pyzmq salt diff --git a/doc/topics/installation/fedora.rst b/doc/topics/installation/fedora.rst index cdfe980f4056..af74afab1562 100644 --- a/doc/topics/installation/fedora.rst +++ b/doc/topics/installation/fedora.rst @@ -11,7 +11,9 @@ makes it a great place to help improve Salt! We are still working to get msgpack packaged for Python 2.6 so Salt is not yet (but nearly) in EPEL 5. In the meantime you can install Salt via our - Fedora People repository:: + Fedora People repository: + + .. code-block:: bash wget -O /etc/yum.repos.d/epel-salt.repo \\ http://repos.fedorapeople.org/repos/herlo/salt/epel-salt.repo @@ -27,7 +29,9 @@ Stable Release Salt is packaged separately for the minion and the master. You'll only need to install the appropriate package for the role you need the machine to play. This -means you're going to want one master and a whole bunch of minions!:: +means you're going to want one master and a whole bunch of minions! + +.. code-block:: bash yum install salt-master yum install salt-minion @@ -50,7 +54,9 @@ configuration paths. By default the Salt master listens on ports 4505 and 4506 on all interfaces (0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the -"interface" directive as seen here:: +"interface" directive as seen here: + +.. code-block:: diff - #interface: 0.0.0.0 + interface: 10.0.0.1 @@ -58,13 +64,19 @@ By default the Salt master listens on ports 4505 and 4506 on all interfaces **Enable the Master** You'll also likely want to activate the Salt Master in systemd, configuring the -Salt Master to start automatically at boot.:: +Salt Master to start automatically at boot. + +.. code-block:: bash systemctl enable salt-master.service +**Start the Master** + Once you've completed all of these steps you're ready to start your Salt Master. You should be able to start your Salt Master now using the command -seen here:: +seen here: + +.. code-block:: bash systemctl start salt-master.service @@ -87,7 +99,9 @@ this you likely can do without any minion configuration at all. If you are not able to update DNS, you'll simply need to update one entry in the configuration file. Using your favorite editor, open the minion -configuration file and update the "master" entry as seen here:: +configuration file and update the "master" entry as seen here: + +.. code-block:: diff - #master: salt + master: 10.0.0.1 @@ -99,13 +113,19 @@ configuration options are covered in another chapter. **Enable the Minion** You'll need to configure the minion to auto-start at boot. You can toggle -that option through systemd.:: +that option through systemd. + +.. code-block:: bash systemctl enable salt-minion.service +**Start the Minion** + Once you've completed all of these steps you're ready to start your Salt Minion. You should be able to start your Salt Minion now using the command -here:: +here: + +.. code-block:: bash systemctl start salt-minion.service @@ -132,28 +152,32 @@ only done through trusted, accepted keys. Before you'll be able to do any remote execution or configuration management you'll need to accept any pending keys on the Master. Run the ``salt-key`` command to -list the keys known to the Salt Master:: +list the keys known to the Salt Master: + +.. code-block:: bash [root@master ~]# salt-key -L Unaccepted Keys: - avon - bodie - bubbles - marlo + alpha + bravo + charlie + delta Accepted Keys: This example shows that the Salt Master is aware of four Minions, but none of the keys have been accepted. To accept the keys and allow the Minions to be -controlled by the Master, again use the ``salt-key`` command:: +controlled by the Master, again use the ``salt-key`` command: + +.. code-block:: bash [root@master ~]# salt-key -A [root@master ~]# salt-key -L Unaccepted Keys: Accepted Keys: - avon - bodie - bubbles - marlo + alpha + bravo + charlie + delta The ``salt-key`` command allows for signing keys individually or in bulk. The example above, using ``-A`` bulk-accepts all pending keys. To accept keys @@ -168,10 +192,12 @@ Whether you have a few or a few-dozen, Salt can help you manage them easily! For final verification, send a test function from your Salt Master to your minions. If all of your minions are properly communicating with your Master, you should "True" responses from each of them. See the example below to send -the ``test.ping`` remote command:: +the ``test.ping`` remote command: + +.. code-block:: bash [root@master ~]# salt '*' test.ping - {'avon': True} + {'alpha': True} Where Do I Go From Here ======================== diff --git a/doc/topics/installation/freebsd.rst b/doc/topics/installation/freebsd.rst index 73b48be4a91a..69f23c42a380 100644 --- a/doc/topics/installation/freebsd.rst +++ b/doc/topics/installation/freebsd.rst @@ -3,11 +3,11 @@ FreeBSD ======= Salt was added to the FreeBSD ports tree Dec 26th, 2011 by Christer Edwards -<christer.edwards@gmail.com>. It has been tested on FreeBSD 8.2 and 9.0 +<christer.edwards@gmail.com>. It has been tested on FreeBSD 7.4, 8.2 and 9.0 releases. Salt is dependent on the following additional ports. These will be installed as -dependencies of the ``sysutils/salt`` port:: +dependencies of the ``sysutils/salt`` port. :: /devel/py-yaml /devel/py-pyzmq @@ -19,7 +19,9 @@ dependencies of the ``sysutils/salt`` port:: Installation ============ -To install Salt from the FreeBSD ports tree, use the command:: +To install Salt from the FreeBSD ports tree, use the command: + +.. code-block:: bash cd /usr/ports/sysutils/salt && make install clean @@ -36,7 +38,9 @@ Master and Salt Minions. The Salt port installs two sample configuration files, ``salt/master.sample`` and ``salt/minion.sample`` (these should be installed in ``/usr/local/etc/``, unless you use a different ``%%PREFIX%%``). You'll need to copy these .sample files into place and -make a few edits. First, copy them into place as seen here:: +make a few edits. First, copy them into place as seen here: + +.. code-block:: bash cp /usr/local/etc/salt/master.sample /usr/local/etc/salt/master cp /usr/local/etc/salt/minion.sample /usr/local/etc/salt/minion @@ -69,17 +73,21 @@ By default the Salt master listens on ports 4505 and 4506 on all interfaces **rc.conf** Last but not least you'll need to activate the Salt Master in your rc.conf -file. Using your favorite editor, open /etc/rc.conf or /etc/rc.conf.local and -add this line. +file. Using your favorite editor, open ``/etc/rc.conf`` or +``/etc/rc.conf.local`` and add this line. .. code-block:: diff + salt_master_enable="YES" +**Start the Master** + Once you've completed all of these steps you're ready to start your Salt Master. The Salt port installs an rc script which should be used to manage your Salt Master. You should be able to start your Salt Master now using the command -seen here:: +seen here: + +.. code-block:: bash service salt_master start @@ -116,8 +124,8 @@ configuration options are covered in another chapter. **rc.conf** Before you're able to start the Salt Minion you'll need to update your rc.conf -file. Using your favorite editor open /etc/rc.conf or /etc/rc.conf.local and -add this line. +file. Using your favorite editor open ``/etc/rc.conf`` or +``/etc/rc.conf.local`` and add this line. .. code-block:: diff @@ -126,7 +134,9 @@ add this line. Once you've completed all of these steps you're ready to start your Salt Minion. The Salt port installs an rc script which should be used to manage your Salt Minion. You should be able to start your Salt Minion now using the command -seen here. :: +seen here. + +.. code-block:: bash service salt_minion start @@ -153,7 +163,9 @@ only done through trusted, accepted keys. Before you'll be able to do any remote execution or state management you'll need to accept any pending keys on the Master. Run the ``salt-key`` command to -list the keys known to the Salt Master:: +list the keys known to the Salt Master: + +.. code-block:: bash [root@master ~]# salt-key -L Unaccepted Keys: @@ -165,7 +177,9 @@ list the keys known to the Salt Master:: This example shows that the Salt Master is aware of four Minions, but none of the keys have been accepted. To accept the keys and allow the Minions to be -controlled by the Master, again use the ``salt-key`` command:: +controlled by the Master, again use the ``salt-key`` command: + +.. code-block:: bash [root@master ~]# salt-key -A [root@master ~]# salt-key -L @@ -202,3 +216,4 @@ able to send remote commands. I'm sure you're eager to learn more about what Salt can do. Depending on the primary way you want to manage your machines you may either want to visit the section regarding Salt States, or the section on Modules. + diff --git a/doc/topics/installation/gentoo.rst b/doc/topics/installation/gentoo.rst index 7cf8b8971efd..513a93eceba7 100644 --- a/doc/topics/installation/gentoo.rst +++ b/doc/topics/installation/gentoo.rst @@ -2,17 +2,23 @@ Gentoo ====== -Salt can be easily installed on Gentoo:: +Salt can be easily installed on Gentoo: + +.. code-block:: bash emerge pyyaml m2crypto pycrypto jinja pyzmq Then download and install from source: -1. Download the latest source tarball from the GitHub downloads directory for - the Salt project: +1. Download the latest source tarball from the `GitHub downloads`_ directory for + the Salt project. + +2. Untar the tarball and run the ``setup.py`` as root: + +.. code-block:: bash -2. Untar the tarball and run the ``setup.py`` as root:: + tar xf salt-<version>.tar.gz + cd salt-<version> + python setup.py install - tar xvf salt-<version>.tar.gz - cd salt-<version> - python2 setup.py install +.. _GitHub downloads: https://github.com/saltstack/salt/downloads From f8dd5a9c8b9cbc5c7f7b0b48d482ca57b0766435 Mon Sep 17 00:00:00 2001 From: Christer Edwards <christer.edwards@gmail.com> Date: Wed, 29 Feb 2012 13:45:46 -0700 Subject: [PATCH 144/598] fixed typo and added note re: FreeBSD file_roots --- doc/index.rst | 2 +- doc/topics/tutorials/states_pt1.rst | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/index.rst b/doc/index.rst index aea492541728..e26ea712c05d 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -16,7 +16,7 @@ Get started with Salt Salt is a **remote execution** and **configuration management** tool. Salt is designed to be secure using **AES encryption** and **public-key -authentication**; incredibly scaleable using an advanced **ZeroMQ** topology; +authentication**; incredibly scalable using an advanced **ZeroMQ** topology; fast and efficient using **msgpack**; and extensible using small and simple **Python** modules. diff --git a/doc/topics/tutorials/states_pt1.rst b/doc/topics/tutorials/states_pt1.rst index 098dee156a7d..df9392aa44f0 100644 --- a/doc/topics/tutorials/states_pt1.rst +++ b/doc/topics/tutorials/states_pt1.rst @@ -28,6 +28,11 @@ uncomment the following lines: base: - /srv/salt +.. note:: + + If you are deploying on FreeBSD via ports, the ``file_roots`` path defaults + to ``/usr/local/etc/salt/states``. + Restart the Salt master in order to pick up this change: .. code-block:: bash From fdfce1b3c3f1c2e7177dcca1133bae92e66c5599 Mon Sep 17 00:00:00 2001 From: Grier Johnson <grierj@gmail.com> Date: Wed, 29 Feb 2012 17:36:03 -0800 Subject: [PATCH 145/598] Clean up multiprocessing processes on SIGTERM First pass at cleaning up the processes spawned by the salt master when run in the foreground via some process minder like runit or daemontools. This current method is a little inelegant as the cleanup code gets pulled into the child processes themselves and you waste a few cycles trying to test the proc objects in the clean_proc() function. That being said, it works and doesn't orphan processes. --- salt/master.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/salt/master.py b/salt/master.py index 789ab3cdb9ad..b552a9bf4ada 100644 --- a/salt/master.py +++ b/salt/master.py @@ -12,6 +12,7 @@ import hashlib import tempfile import datetime +import signal import multiprocessing import subprocess @@ -49,6 +50,26 @@ def prep_jid(opts, load): return prep_jid(opts['cachedir'], load) return jid +def clean_proc(proc, wait_for_kill=1): + ''' + Generic method for cleaning up multiprocessing procs + ''' + # NoneType and other fun stuff need not apply + if not proc: + return + try: + waited = 0 + while proc.is_alive(): + proc.terminate() + waited += 1 + time.sleep(1) + if proc.is_alive() and (waited >= wait_for_kill): + log.error(('Process did not die with terminate(): {0}' + .format(proc.pid))) + os.kill(signal.SIGKILL, proc.pid) + except Exception as e: + log.debug(e) + class SMaster(object): ''' @@ -121,7 +142,9 @@ def start(self): Turn on the master server components ''' log.warn('Starting the Salt Master') - multiprocessing.Process(target=self._clear_old_jobs).start() + clear_old_jobs_proc = multiprocessing.Process( + target=self._clear_old_jobs) + clear_old_jobs_proc.start() aes_funcs = AESFuncs(self.opts, self.crypticle) clear_funcs = ClearFuncs( self.opts, @@ -137,12 +160,31 @@ def start(self): clear_funcs) reqserv.start_publisher() + def sigterm_clean(signum, frame): + ''' + Cleaner method for stoping multiprocessing processes when a SIGTERM + is encountered. This is required when running a salt master under + a process minder like daemontools + ''' + mypid = os.getpid() + log.warn(('Caught signal {0}, stopping the Salt Master' + .format(signum))) + clean_proc(clear_old_jobs_proc) + clean_proc(reqserv.publisher) + for proc in reqserv.work_procs: + clean_proc(proc) + raise SystemExit + + signal.signal(signal.SIGTERM, sigterm_clean) + try: reqserv.run() except KeyboardInterrupt: # Shut the master down gracefully on SIGINT log.warn('Stopping the Salt Master') raise SystemExit('\nExiting on Ctrl-c') + finally: + raise SystemExit('Salt Master Stopped') class Publisher(multiprocessing.Process): @@ -207,15 +249,19 @@ def __bind(self): ''' log.info('Setting up the master communication server') self.clients.bind(self.uri) + self.work_procs = [] for ind in range(int(self.opts['worker_threads'])): - log.info('Starting Salt worker process {0}'.format(ind)) - MWorker(self.opts, + self.work_procs.append(MWorker(self.opts, self.master_key, self.key, self.crypticle, self.aes_funcs, - self.clear_funcs).start() + self.clear_funcs)) + + for ind, proc in enumerate(self.work_procs): + log.info('Starting Salt worker process {0}'.format(ind)) + proc.start() self.workers.bind(self.w_uri) From d4437c075f03779d5d6852c68f52928c6ea2b4f5 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 29 Feb 2012 20:24:19 -0700 Subject: [PATCH 146/598] Initial support for openSUSE's zypper --- salt/modules/zypper.py | 224 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 salt/modules/zypper.py diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py new file mode 100644 index 000000000000..4886831626d6 --- /dev/null +++ b/salt/modules/zypper.py @@ -0,0 +1,224 @@ +''' +Package support for openSUSE via the zypper package manager +''' + + +def __virtual__(): + ''' + Set the virtual pkg module if the os is openSUSE + ''' + return 'pkg' if __grains__['os'] == 'openSUSE' else False + + +def _list_removed(old, new): + ''' + List the packages which have been removed between the two package objects + ''' + pkgs = [] + for pkg in old: + if pkg not in new: + pkgs.append(pkg) + return pkgs + + +def _available_versions(): + ''' + The available versions of packages + ''' + cmd = 'zypper packages -i' + ret = {} + out = __salt__['cmd.run'](cmd).split('\n') + for line in out: + if not line: + continue + if '|' not in line: + continue + comps = [] + for comp in line.split('|'): + comps.append(comp.strip()) + if comps[0] == 'v': + ret[comps[2]] = comps[3] + return ret + +def available_version(name): + ''' + Return the available version of a given package + + CLI Example:: + + salt '*' pkg.available_version <package name> + ''' + avail = _available_versions() + return avail.get(name, '') + +def version(name): + ''' + Returns a version if the package is installed, else returns an empty string + + CLI Example:: + + salt '*' pkg.version <package name> + ''' + pkgs = list_pkgs() + if name in pkgs: + return pkgs[name] + else: + return '' + + +def list_pkgs(): + ''' + List the packages currently installed as a dict:: + + {'<package_name>': '<version>'} + + CLI Example:: + + salt '*' pkg.list_pkgs + ''' + cmd = 'zypper packages -i' + ret = {} + out = __salt__['cmd.run'](cmd).split('\n') + for line in out: + if not line: + continue + if '|' not in line: + continue + comps = [] + for comp in line.split('|'): + comps.append(comp.strip()) + if comps[0] == 'i': + ret[comps[2]] = comps[3] + return ret + + +def refresh_db(): + ''' + Just run a ``zypper refresh``, return a dict:: + + {'<database name>': Bool} + + CLI Example:: + + salt '*' pkg.refresh_db + ''' + cmd = 'zypper refresh' + ret = {} + out = __salt__['cmd.run'](cmd).split('\n') + for line in out: + if not line: + continue + if line.strip().startswith('Repository'): + key = line.split("'")[1].strip() + if 'is up to date' in line: + ret[key] = False + elif line.strip().startswith('Building'): + key = line.split("'")[1].strip() + if 'done' in line: + ret[key] = True + return ret + + +def install(name, refresh=False, **kwargs): + ''' + Install the passed package, add refresh=True to install with an -Sy + + Return a dict containing the new package names and versions:: + + {'<package>': {'old': '<old-version>', + 'new': '<new-version>']} + + CLI Example:: + + salt '*' pkg.install <package name> + ''' + old = list_pkgs() + cmd = 'zypper -n install -l {0}'.format(name) + if refresh: + refresh_db() + __salt__['cmd.retcode'](cmd) + new = list_pkgs() + pkgs = {} + for npkg in new: + if npkg in old: + if old[npkg] == new[npkg]: + # no change in the package + continue + else: + # the package was here before and the version has changed + pkgs[npkg] = {'old': old[npkg], + 'new': new[npkg]} + else: + # the package is freshly installed + pkgs[npkg] = {'old': '', + 'new': new[npkg]} + return pkgs + + +def upgrade(): + ''' + Run a full system upgrade, a zypper upgrade + + Return a dict containing the new package names and versions:: + + {'<package>': {'old': '<old-version>', + 'new': '<new-version>']} + + CLI Example:: + + salt '*' pkg.upgrade + ''' + old = list_pkgs() + cmd = 'zypper -n up -l' + __salt__['cmd.retcode'](cmd) + new = list_pkgs() + pkgs = {} + for npkg in new: + if npkg in old: + if old[npkg] == new[npkg]: + # no change in the package + continue + else: + # the package was here before and the version has changed + pkgs[npkg] = {'old': old[npkg], + 'new': new[npkg]} + else: + # the package is freshly installed + pkgs[npkg] = {'old': '', + 'new': new[npkg]} + return pkgs + + +def remove(name): + ''' + Remove a single package with ``zypper remove`` + + Return a list containing the removed packages. + + CLI Example:: + + salt '*' pkg.remove <package name> + ''' + old = list_pkgs() + cmd = 'zypper -n remove {0}'.format(name) + __salt__['cmd.retcode'](cmd) + new = list_pkgs() + return _list_removed(old, new) + + +def purge(name): + ''' + Recursively remove a package and all dependencies which were installed + with it, this will call a ``zypper remove -u`` + + Return a list containing the removed packages. + + CLI Example:: + + salt '*' pkg.purge <package name> + ''' + old = list_pkgs() + cmd = 'zypper -n remove -u {0}'.format(name) + __salt__['cmd.retcode'](cmd) + new = list_pkgs() + return _list_removed(old, new) From 6e7253571584dbc1f3ab7a0608b9e6fd5abcf3de Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 29 Feb 2012 20:26:06 -0700 Subject: [PATCH 147/598] Add openSUSE to use systemd --- salt/modules/systemd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/modules/systemd.py b/salt/modules/systemd.py index a95f1b375c6d..51075a3bd36b 100644 --- a/salt/modules/systemd.py +++ b/salt/modules/systemd.py @@ -11,6 +11,8 @@ def __virtual__(): ''' if __grains__['os'] == 'Fedora' and __grains__['osrelease'] > 15: return 'service' + elif __grains__['os'] == 'openSUSE': + return 'service' return False From 97c5cb0312d7b093752376a373cc3773fcf44f34 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 29 Feb 2012 21:05:07 -0700 Subject: [PATCH 148/598] Add SunOS to the basic service module --- salt/modules/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/service.py b/salt/modules/service.py index 5aee6db4ca21..103a6eea0bf2 100644 --- a/salt/modules/service.py +++ b/salt/modules/service.py @@ -13,6 +13,7 @@ 'Ubuntu': '/etc/init.d', 'Gentoo': '/etc/init.d', 'CentOS': '/etc/init.d', + 'SunOS': '/etc/init.d', } def __virtual__(): From b25fc9e7f1008382e5a26f81c3cc76848abe8160 Mon Sep 17 00:00:00 2001 From: Erik Johnson <palehose@gmail.com> Date: Wed, 29 Feb 2012 23:29:57 -0600 Subject: [PATCH 149/598] add upgrade_available() --- salt/modules/pacman.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/modules/pacman.py b/salt/modules/pacman.py index 163f776dd9aa..2f1d6fb385db 100644 --- a/salt/modules/pacman.py +++ b/salt/modules/pacman.py @@ -33,6 +33,18 @@ def available_version(name): return __salt__['cmd.run']('pacman -Sp --print-format %v {0}'.format(name)) +def upgrade_available(name): + ''' + Check whether or not an upgrade is available for a given package + + CLI Example:: + + salt '*' pkg.upgrade_available <package name> + ''' + return name in __salt__['cmd.run']( + 'pacman -Spu --print-format %n | egrep "^\S+$"').split() + + def version(name): ''' Returns a version if the package is installed, else returns an empty string From bf953e8e2bee92d8e481bc9dd0f9cbd021bc2c1e Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Wed, 29 Feb 2012 22:38:33 -0700 Subject: [PATCH 150/598] Add nslookup() method --- salt/modules/win_network.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/modules/win_network.py b/salt/modules/win_network.py index 8c6c687945a2..d70fc532801f 100644 --- a/salt/modules/win_network.py +++ b/salt/modules/win_network.py @@ -122,9 +122,33 @@ def traceroute(host): return ret +def nslookup(host): + ''' + Query DNS for information about a domain or ip address + + CLI Example:: + + salt '*' network.nslookup archlinux.org + ''' + ret = [] + cmd = 'nslookup %s' % _sanitize_host(host) + lines = __salt__['cmd.run'](cmd).split('\n') + for line in lines: + if line.startswith('Non-authoritative'): + continue + if ":" in line: + comps = line.split(":") + ret.append({comps[0].strip(): comps[1].strip()}) + + return ret + + + def dig(host): ''' Performs a DNS lookup with dig + + Note: dig must be installed on the Windows minion CLI Example:: From 5aab2de7ce287f53677980ebbb243c5ed634214f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 29 Feb 2012 23:01:32 -0700 Subject: [PATCH 151/598] Fix trace in file.managed when no source is given and there is no parent dir --- salt/states/file.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 24ee37b28450..97065158fe86 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -697,24 +697,31 @@ def managed(name, ret['result'] = False ret['comment'] = 'Source file {0} not found'.format(source) return ret - if not __opts__['test']: - if not os.path.isdir(os.path.dirname(name)): - if makedirs: - _makedirs(name) - else: - ret['result'] = False - ret['comment'] = 'Parent directory not present' - __clean_tmp(sfn) - return ret + if not os.path.isdir(os.path.dirname(name)): + if makedirs: + _makedirs(name) + else: + ret['result'] = False + ret['comment'] = 'Parent directory not present' + __clean_tmp(sfn) + return ret else: - ret['changes']['new'] = 'file {0} created'.format(name) - ret['comment'] = 'Empty file' - # Create the file, user-rw-only if mode will be set - if mode: - cumask = os.umask(384) - open(name, 'a').close() - if mode: - os.umask(cumask) + if not os.path.isdir(os.path.dirname(name)): + if makedirs: + _makedirs(name) + else: + ret['result'] = False + ret['comment'] = 'Parent directory not present' + __clean_tmp(sfn) + return ret + # Create the file, user-rw-only if mode will be set + if mode: + cumask = os.umask(384) + open(name, 'a+').close() + if mode: + os.umask(cumask) + ret['changes']['new'] = 'file {0} created'.format(name) + ret['comment'] = 'Empty file' # Check permissions perms = {} perms['luser'] = __salt__['file.get_user'](name) From 7e2a31a553ba9461a63220f4dafc310620f245ed Mon Sep 17 00:00:00 2001 From: Erik Johnson <palehose@gmail.com> Date: Thu, 1 Mar 2012 10:57:24 -0600 Subject: [PATCH 152/598] add upgrade_available() --- salt/modules/yumpkg.py | 14 ++++++++++++++ salt/modules/yumpkg5.py | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 29faac74954f..9578080eede5 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -97,6 +97,20 @@ def available_version(name): return list(set(versions_list))[0] +def upgrade_available(name): + ''' + Check whether or not an upgrade is available for a given package + + CLI Example:: + + salt '*' pkg.upgrade_available <package name> + ''' + if available_version(name): + return True + else: + return False + + def version(name): ''' Returns a version if the package is installed, else returns an empty string diff --git a/salt/modules/yumpkg5.py b/salt/modules/yumpkg5.py index 50e523ec82b6..fdd6565a7767 100644 --- a/salt/modules/yumpkg5.py +++ b/salt/modules/yumpkg5.py @@ -73,6 +73,17 @@ def available_version(name): return out[0].version if out else '' +def upgrade_available(name): + ''' + Check whether or not an upgrade is available for a given package + + CLI Example:: + + salt '*' pkg.upgrade_available <package name> + ''' + return available_version(name) != '' + + def version(name): ''' Returns a version if the package is installed, else returns an empty string From 88c2983de22be00e94c0203cd981c3454853ba40 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 1 Mar 2012 10:16:20 -0700 Subject: [PATCH 153/598] Add dns checks in self healing --- salt/minion.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index ab162312ab8d..cd4744169cfc 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -13,6 +13,7 @@ import re import shutil import string +import socket import tempfile import threading import time @@ -25,7 +26,8 @@ # Import salt libs from salt.exceptions import AuthenticationError, MinionError, \ - CommandExecutionError, CommandNotFoundError, SaltInvocationError + CommandExecutionError, CommandNotFoundError, SaltInvocationError, \ + SaltMasterError import salt.client import salt.crypt import salt.loader @@ -58,6 +60,25 @@ def get_proc_dir(cachedir): return fn_ +def safe_dns_check(addr): + ''' + Return the ip resolved by dns, but do not exit on failure, only raise an + exception. + ''' + try: + socket.inet_aton(addr) + except socket.error: + # Not a valid ip adder, check DNS + try: + addr = socket.gethostbyname(addr) + except socket.gaierror: + err = ('This master address: {0} was previously resolvable but ' + 'now fails to resolve! The previously resolved ip addr ' + 'will continue to be used').format(addr) + log.error(err) + raise SaltMasterError + + class SMinion(object): ''' Create an object that has loaded all of the minion module functions, @@ -430,6 +451,14 @@ def tune_in(self): if time.time() - last > self.opts['sub_timeout']: # It has been a while since the last command, make sure # the connection is fresh by reconnecting + if self.opts['dns_check']: + try: + # Verify that the dns entry has not changed + self.opts['master_ip'] = safe_dns_check() + except SaltMasterError: + # Failed to update the dns, keep the old addr + pass + self.passive_refresh() socket.close() socket = context.socket(zmq.SUB) socket.setsockopt(zmq.SUBSCRIBE, '') @@ -437,7 +466,6 @@ def tune_in(self): last = time.time() time.sleep(0.05) multiprocessing.active_children() - self.passive_refresh() else: while True: payload = None From 10cce187a4bcbe6af8018734732c19bf33fcfbee Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 1 Mar 2012 10:17:16 -0700 Subject: [PATCH 154/598] Add salt master error --- salt/exceptions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/exceptions.py b/salt/exceptions.py index 79ebe28921df..0f2e1d7fb59d 100644 --- a/salt/exceptions.py +++ b/salt/exceptions.py @@ -14,6 +14,12 @@ class SaltClientError(SaltException): ''' pass +class SaltMasterError(SaltException): + ''' + Problem reading the master root key + ''' + pass + class AuthenticationError(SaltException): ''' If sha256 signature fails during decryption From 3a69744cd52b2dcdab15b2e399f99217a462278c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 1 Mar 2012 10:20:34 -0700 Subject: [PATCH 155/598] change error type for new dns checks --- salt/minion.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index cd4744169cfc..2b4f2de41860 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -26,8 +26,7 @@ # Import salt libs from salt.exceptions import AuthenticationError, MinionError, \ - CommandExecutionError, CommandNotFoundError, SaltInvocationError, \ - SaltMasterError + CommandExecutionError, CommandNotFoundError, SaltInvocationError import salt.client import salt.crypt import salt.loader @@ -76,7 +75,7 @@ def safe_dns_check(addr): 'now fails to resolve! The previously resolved ip addr ' 'will continue to be used').format(addr) log.error(err) - raise SaltMasterError + raise SaltClientError class SMinion(object): @@ -455,7 +454,7 @@ def tune_in(self): try: # Verify that the dns entry has not changed self.opts['master_ip'] = safe_dns_check() - except SaltMasterError: + except SaltClientError: # Failed to update the dns, keep the old addr pass self.passive_refresh() From 140b36b8a5bb825d65c57443cb553b6b32e23dff Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 1 Mar 2012 10:24:19 -0700 Subject: [PATCH 156/598] Add dns)_check to the minion config --- conf/minion.template | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/minion.template b/conf/minion.template index 59c5700d1cb7..2ae3bae13d0d 100644 --- a/conf/minion.template +++ b/conf/minion.template @@ -44,6 +44,10 @@ # seconds, between those reconnection attempts. #acceptance_wait_time = 10 +# When healing a dns_check is run, this is to make sure that the originally +# resolved dns has not changed, if this is something that does not happen in +# your environment then set this value to False. +#dns_check: True ##### Minion module management ##### From 2ef9c58e97813f62d8dd1ebdfc8dc6f1fb20930a Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Thu, 1 Mar 2012 09:48:30 -0800 Subject: [PATCH 157/598] Add configuration for key logfile path and ensure it uses root_dir --- salt/cli/__init__.py | 8 ++++---- salt/config.py | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 945be87cf416..d7dfcbe16135 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -529,8 +529,8 @@ def __parse(self): action='store_true', help='Supress output') - parser.add_option('--logfile', - dest='logfile', + parser.add_option('--key-logfile', + dest='key_logfile', default='/var/log/salt/key.log', help=('Send all output to a file. ' 'Default is /var/log/salt/key.log')) @@ -566,7 +566,7 @@ def __parse(self): opts = {} opts['quiet'] = options.quiet - opts['logfile'] = options.logfile + opts['key_logfile'] = options.key_logfile # I decided to always set this to info, since it really all is info or # error. opts['loglevel'] = 'info' @@ -596,7 +596,7 @@ def run(self): Execute saltkey ''' import salt.log - salt.log.setup_logfile_logger(self.opts['logfile'], + salt.log.setup_logfile_logger(self.opts['key_logfile'], self.opts['loglevel']) key = salt.cli.key.Key(self.opts) key.run() diff --git a/salt/config.py b/salt/config.py index 035f79af6999..83c330a34b76 100644 --- a/salt/config.py +++ b/salt/config.py @@ -174,8 +174,7 @@ def minion_config(path): opts['grains'] = salt.loader.grains(opts) # Prepend root_dir to other paths - prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file']) - + prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', 'key_logfile']) return opts @@ -214,6 +213,7 @@ def master_config(path): 'cluster_mode': 'paranoid', 'serial': 'msgpack', 'nodegroups': {}, + 'key_logfile': '/var/log/salt/key.log', } load_config(opts, path, 'SALT_MASTER_CONFIG') @@ -224,7 +224,8 @@ def master_config(path): opts['aes'] = salt.crypt.Crypticle.generate_key_string() # Prepend root_dir to other paths - prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', 'sock_dir']) + prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', + 'sock_dir', 'key_logfile']) # Enabling open mode requires that the value be set to True, and # nothing else! From 2539e1e735f50098d3562b3a43fe59eccba8e8be Mon Sep 17 00:00:00 2001 From: Matthew Nicholson <matthew_nicholson@harvard.edu> Date: Thu, 1 Mar 2012 13:23:48 -0500 Subject: [PATCH 158/598] new section on open file limits --- doc/topics/troubleshooting/index.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/topics/troubleshooting/index.rst b/doc/topics/troubleshooting/index.rst index 571a633818ac..d24fef8dffa7 100644 --- a/doc/topics/troubleshooting/index.rst +++ b/doc/topics/troubleshooting/index.rst @@ -30,4 +30,25 @@ When developing the state tree it is geenrally recommended to invoke state.highstate with salt-call, this displays a great deal more information about the highstate execution then if it is called remotely. +Too many open files +=================== +The salt-master needs at least 2 sockets per host that connects to it, one for +the Publisher and one for response port. Thus, large installations may upon +scaling up the number of minions accessing a given master, encounter: + + 12:45:29,289 [salt.master ][INFO ] Starting Salt worker process 38 + Too many open files + sock != -1 (tcp_listener.cpp:335) + +The solution to this would be to check the number of files allowed to be +opened by the user running salt-master (root by default): + + [root@salt-master ~]# ulimit -n + 1024 + +And modify that value to be at least equal to the number of minions x 2. +This setting can be changed in limits.conf as the nofile value(s), +and activated upon new a login of the specified user. + +So, an environment with 1800 minions, would need 1800 x 2 = 3600 as a minimum From 12bed5ff07b38828a156aa43b65451500e82fb55 Mon Sep 17 00:00:00 2001 From: Grier Johnson <grierj@gmail.com> Date: Thu, 1 Mar 2012 11:23:05 -0800 Subject: [PATCH 159/598] Updating multiprocessing child clean-up Fixing minor issues in the first pass for the multiprocessing child clean-up code. --- salt/__init__.py | 5 ++++- salt/master.py | 22 ++++++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/salt/__init__.py b/salt/__init__.py index 515180276fc0..06801534cde6 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -172,7 +172,10 @@ def start(self): import salt.utils salt.utils.daemonize() set_pidfile(self.cli['pidfile']) - master.start() + try: + master.start() + except salt.master.MasterExit: + sys.exit() class Minion(object): diff --git a/salt/master.py b/salt/master.py index b552a9bf4ada..8a3e08bbf7d0 100644 --- a/salt/master.py +++ b/salt/master.py @@ -67,9 +67,17 @@ def clean_proc(proc, wait_for_kill=1): log.error(('Process did not die with terminate(): {0}' .format(proc.pid))) os.kill(signal.SIGKILL, proc.pid) - except Exception as e: - log.debug(e) + except (AssertionError, AttributeError) as e: + # Catch AssertionError when the proc is evaluated inside the child + # Catch AttributeError when the process dies between proc.is_alive() + # and proc.terminate() and turns into a NoneType + pass +class MasterExit(SystemExit): + ''' + Named exit exception for the master process exiting + ''' + pass class SMaster(object): ''' @@ -162,9 +170,9 @@ def start(self): def sigterm_clean(signum, frame): ''' - Cleaner method for stoping multiprocessing processes when a SIGTERM - is encountered. This is required when running a salt master under - a process minder like daemontools + Cleaner method for stopping multiprocessing processes when a + SIGTERM is encountered. This is required when running a salt + master under a process minder like daemontools ''' mypid = os.getpid() log.warn(('Caught signal {0}, stopping the Salt Master' @@ -173,7 +181,7 @@ def sigterm_clean(signum, frame): clean_proc(reqserv.publisher) for proc in reqserv.work_procs: clean_proc(proc) - raise SystemExit + raise MasterExit signal.signal(signal.SIGTERM, sigterm_clean) @@ -183,8 +191,6 @@ def sigterm_clean(signum, frame): # Shut the master down gracefully on SIGINT log.warn('Stopping the Salt Master') raise SystemExit('\nExiting on Ctrl-c') - finally: - raise SystemExit('Salt Master Stopped') class Publisher(multiprocessing.Process): From f1f372776b80d61a88edd685a8645ed006dab20e Mon Sep 17 00:00:00 2001 From: Grier Johnson <grierj@gmail.com> Date: Thu, 1 Mar 2012 13:34:20 -0800 Subject: [PATCH 160/598] Faster Master Clean-up Time Change the sleep time to 1/10th of a second and check on the process 10 times before resorting to SIGKILL. This results in a considerable speed-up in shutdown time. --- salt/master.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/master.py b/salt/master.py index 8a3e08bbf7d0..64a532b7b7db 100644 --- a/salt/master.py +++ b/salt/master.py @@ -50,7 +50,7 @@ def prep_jid(opts, load): return prep_jid(opts['cachedir'], load) return jid -def clean_proc(proc, wait_for_kill=1): +def clean_proc(proc, wait_for_kill=10): ''' Generic method for cleaning up multiprocessing procs ''' @@ -62,7 +62,7 @@ def clean_proc(proc, wait_for_kill=1): while proc.is_alive(): proc.terminate() waited += 1 - time.sleep(1) + time.sleep(0.1) if proc.is_alive() and (waited >= wait_for_kill): log.error(('Process did not die with terminate(): {0}' .format(proc.pid))) From bfd2cb24b114b0a282f6996c414a540c7af591ec Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 1 Mar 2012 17:15:40 -0700 Subject: [PATCH 161/598] Add non-blocking iterative cmd method to local client --- salt/client.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/salt/client.py b/salt/client.py index 31feb5e103a8..7bcfe53269e0 100644 --- a/salt/client.py +++ b/salt/client.py @@ -175,6 +175,35 @@ def cmd( return self.get_returns(pub_data['jid'], pub_data['minions'], timeout) def cmd_iter( + self, + tgt, + fun, + arg=(), + timeout=None, + expr_form='glob', + ret=''): + ''' + Execute a salt command and return + ''' + if timeout is None: + timeout = self.opts['timeout'] + jid = prep_jid(self.opts['cachedir']) + pub_data = self.pub( + tgt, + fun, + arg, + expr_form, + ret, + jid=jid, + timeout=timeout) + for fn_ret in self.get_iter_returns(pub_data['jid'], + pub_data['minions'], + timeout): + if not fn_ret: + continue + yield fn_ret + + def cmd_iter_no_block( self, tgt, fun, @@ -276,6 +305,7 @@ def get_iter_returns(self, jid, minions, timeout=None): # No minions have replied within the specified global timeout, # return an empty dict break + yield None time.sleep(0.02) def get_returns(self, jid, minions, timeout=None): From 35eb777c124295d216de6a003f912feea7338ccb Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 1 Mar 2012 17:16:12 -0700 Subject: [PATCH 162/598] Add initial --batch option Yes, there has been some disagreement on the name, I care a lot less about why people dislike --batch and care a lot more about arguments for something else to call it. --- salt/cli/__init__.py | 13 ++++- salt/cli/batch.py | 117 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 salt/cli/batch.py diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 945be87cf416..6cdcca906021 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -11,6 +11,7 @@ import salt.cli.caller import salt.cli.cp import salt.cli.key +import salt.cli.batch import salt.client import salt.output import salt.runner @@ -50,6 +51,13 @@ def __parse(self): dest='iter_', action='store_true', help='Return the data from minions as the data is returned') + parser.add_option('-b', + '--batch', + default='', + dest='batch', + help=('Execute the salt job in batch mode, pass either the ' + 'number of minions to batch at a time, or the ' + 'percentage of minions to have running')) parser.add_option('-E', '--pcre', default=False, @@ -163,6 +171,7 @@ def __parse(self): if not options.timeout is None: opts['timeout'] = int(options.timeout) opts['iter'] = options.iter_ + opts['batch'] = options.batch opts['pcre'] = options.pcre opts['list'] = options.list_ opts['grain'] = options.grain @@ -242,7 +251,9 @@ def run(self): print 'Return data for job {0}:'.format(jid) printout(ret[jid]) print '' - + elif self.opts['batch']: + batch = salt.cli.batch.Batch(self.opts) + batch.run() else: if not 'timeout' in self.opts: self.opts['timeout'] = local.opts['timeout'] diff --git a/salt/cli/batch.py b/salt/cli/batch.py new file mode 100644 index 000000000000..f535ad39f100 --- /dev/null +++ b/salt/cli/batch.py @@ -0,0 +1,117 @@ +''' +Execute batch runs +''' +# Import Python libs +import time +import copy + +# Import Salt libs +import salt.client + +class Batch(object): + ''' + Manage the execution of batch runs + ''' + def __init__(self, opts): + self.opts = opts + self.local = salt.client.LocalClient(opts['conf_file']) + self.minions = self.__gather_minions() + + def __gather_minions(self): + ''' + Return a list of minions to use for the batch run + ''' + args = [self.opts['tgt'], + 'test.ping', + [], + 1, + ] + if self.opts['pcre']: + args.append('pcre') + elif self.opts['list']: + args.append('list') + elif self.opts['grain']: + args.append('grain') + elif self.opts['grain_pcre']: + args.append('grain_pcre') + elif self.opts['exsel']: + args.append('exsel') + elif self.opts['nodegroup']: + args.append('nodegroup') + elif self.opts['compound']: + args.append('compound') + else: + args.append('glob') + + fret = [] + for ret in self.local.cmd_iter(*args): + for minion in ret: + print '{0} Detected for this batch run'.format(minion) + fret.append(minion) + return sorted(fret) + + def get_bnum(self): + ''' + Return the active number of minions to maintain + ''' + if self.opts['batch'].startswith('%'): + return int(float(self.opts['batch']) * snum) + return int(self.opts['batch']) + + def run(self): + ''' + Execute the batch run + ''' + args = [[], + self.opts['fun'], + self.opts['arg'], + 9999999999, + 'list', + ] + bnum = self.get_bnum() + to_run = copy.deepcopy(self.minions) + active = [] + ret = {} + iters = [] + # Itterate while we still have things to execute + while len(ret) < len(self.minions): + next_ = [] + if len(to_run) <= bnum and not active: + # last bit of them, add them all to next iterator + while to_run: + next_.append(to_run.pop()) + else: + for ind in range(bnum - len(active)): + if to_run: + next_.append(to_run.pop()) + active += next_ + args[0] = next_ + if next_: + print '\nExecuting run on {0}\n'.format(next_) + iters.append( + self.local.cmd_iter_no_block(*args)) + else: + time.sleep(0.02) + parts = {} + for queue in iters: + try: + # Gather returns until we get to the bottom + ncnt = 0 + while True: + part = queue.next() + if part is None: + time.sleep(0.01) + ncnt += 1 + if ncnt > 5: + break + continue + print part + parts.update(part) + except StopIteration: + # remove the iter, it is done + pass + for minion, data in parts.items(): + active.remove(minion) + ret[minion] = data['ret'] + #print minion + #print ret[minion] From eae7bea4f609b151b6276c6207fbc41b4e565b1f Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Thu, 1 Mar 2012 16:17:03 -0800 Subject: [PATCH 163/598] Add requirements.txt to sdist and allow setup.py to use setuptools if detected in environment --- MANIFEST.in | 1 + setup.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 7c985df0444a..7802228b732b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include AUTHORS include LICENSE include README.rst +include requirements.txt \ No newline at end of file diff --git a/setup.py b/setup.py index c88f81cc76eb..3370e8bad710 100755 --- a/setup.py +++ b/setup.py @@ -11,6 +11,9 @@ from distutils.cmd import Command from distutils.sysconfig import get_python_lib, PREFIX +if os.environ.get('VIRTUAL_ENV'): + from setuptools import setup + execfile('salt/version.py') class TestCommand(Command): @@ -48,6 +51,10 @@ def run(self): libraries = ['ws2_32'] if sys.platform == 'win32' else [] +requirements='' +with open('requirements.txt') as f: + requirements = f.read() + setup( name=NAME, version=VER, @@ -108,4 +115,5 @@ def run(self): ['doc/man/salt.7', ]), ], + install_requires=requirements, ) From df09f4e52b446cc4e15af87b57c11321b7a9b054 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 1 Mar 2012 17:35:19 -0700 Subject: [PATCH 164/598] Make % management in batch more flexible --- salt/cli/batch.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/cli/batch.py b/salt/cli/batch.py index f535ad39f100..a5c61cb9a958 100644 --- a/salt/cli/batch.py +++ b/salt/cli/batch.py @@ -54,8 +54,14 @@ def get_bnum(self): ''' Return the active number of minions to maintain ''' - if self.opts['batch'].startswith('%'): - return int(float(self.opts['batch']) * snum) + try: + if self.opts['batch'].startswith('%'): + return int(float(self.opts['batch'][1:]) / 100.0 * snum) + elif self.opts['batch'].endswith('%'): + return int(float(self.opts['batch'][:-1]) / 100.0 * snum) + except ValueError: + print ('Invalid batch data sent: {0}\nData must be in the form' + 'of %10, 10% or 3').format(self.opts['batch']) return int(self.opts['batch']) def run(self): From cd9eb21dc00c70425da11b51a44bf40481dedddd Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 1 Mar 2012 17:39:53 -0700 Subject: [PATCH 165/598] add snum definition to figuring percentages --- salt/cli/batch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/cli/batch.py b/salt/cli/batch.py index a5c61cb9a958..cff73fe9a370 100644 --- a/salt/cli/batch.py +++ b/salt/cli/batch.py @@ -54,6 +54,7 @@ def get_bnum(self): ''' Return the active number of minions to maintain ''' + snum = len(self.minions) try: if self.opts['batch'].startswith('%'): return int(float(self.opts['batch'][1:]) / 100.0 * snum) From 005086a95736e2baa9e29b683625236d816cf394 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 1 Mar 2012 20:17:20 -0700 Subject: [PATCH 166/598] Add printout command to output module --- salt/output.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/salt/output.py b/salt/output.py index be49fa298e02..fad9b64f283a 100644 --- a/salt/output.py +++ b/salt/output.py @@ -2,9 +2,11 @@ A simple way of setting the output format for data from modules ''' +# Import Python libs import json import pprint +# Import third party libs import yaml try: yaml.Loader = yaml.CLoader @@ -12,16 +14,34 @@ except: pass +# Import Salt libs import salt.utils +from salt.exceptions import SaltException __all__ = ('get_outputter',) -def remove_colors(): +def display_output(ret, out, opts): ''' - Access all of the utility colors and change them to empty strings + Display the output of a command in the terminal ''' - pass + if isinstance(ret, list) or isinstance(ret, dict): + if opts['raw_out']: + printout = get_outputter('raw') + elif opts['json_out']: + printout = get_outputter('json') + elif opts['txt_out']: + printout = get_outputter('txt') + elif opts['yaml_out']: + printout = get_outputter('yaml') + elif out: + printout = get_outputter(out) + else: + printout = get_outputter(None) + # Pretty print any salt exceptions + elif isinstance(ret, SaltException): + printout = get_outputter("txt") + printout(ret) class Outputter(object): From 33ca2d22ed9e053ab5264b6c888549f3e3adb13d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 1 Mar 2012 20:18:00 -0700 Subject: [PATCH 167/598] Add outputter printing to the batch system --- salt/cli/batch.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/cli/batch.py b/salt/cli/batch.py index cff73fe9a370..b8f69dede7b9 100644 --- a/salt/cli/batch.py +++ b/salt/cli/batch.py @@ -7,6 +7,7 @@ # Import Salt libs import salt.client +import salt.output class Batch(object): ''' @@ -112,7 +113,6 @@ def run(self): if ncnt > 5: break continue - print part parts.update(part) except StopIteration: # remove the iter, it is done @@ -120,5 +120,13 @@ def run(self): for minion, data in parts.items(): active.remove(minion) ret[minion] = data['ret'] - #print minion - #print ret[minion] + data[minion] = data.pop('ret') + if 'out' in data: + out = data.pop('out') + else: + out = None + salt.output.display_output( + data, + out, + self.opts) + return ret From 8c375568f4bc94b37f65de9210923187618c7d93 Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Thu, 1 Mar 2012 21:32:13 -0700 Subject: [PATCH 168/598] Fix errors regarding dns_check Added dns_check to the default opt config for the minion. Added SaltClientError to the imports Updated safe_dns_check() to return the ip address if the dns is good. Fixed an instance of safe_dns_check() to use the master config as arg. --- salt/config.py | 1 + salt/minion.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/config.py b/salt/config.py index 83c330a34b76..e9cd4a39933d 100644 --- a/salt/config.py +++ b/salt/config.py @@ -151,6 +151,7 @@ def minion_config(path): 'cython_enable': False, 'state_verbose': False, 'acceptance_wait_time': 10, + 'dns_check': True, } load_config(opts, path, 'SALT_MINION_CONFIG') diff --git a/salt/minion.py b/salt/minion.py index 2b4f2de41860..426f7d50fd73 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -26,7 +26,8 @@ # Import salt libs from salt.exceptions import AuthenticationError, MinionError, \ - CommandExecutionError, CommandNotFoundError, SaltInvocationError + CommandExecutionError, CommandNotFoundError, SaltInvocationError, \ + SaltClientError import salt.client import salt.crypt import salt.loader @@ -76,6 +77,7 @@ def safe_dns_check(addr): 'will continue to be used').format(addr) log.error(err) raise SaltClientError + return addr class SMinion(object): @@ -453,7 +455,7 @@ def tune_in(self): if self.opts['dns_check']: try: # Verify that the dns entry has not changed - self.opts['master_ip'] = safe_dns_check() + self.opts['master_ip'] = safe_dns_check(self.opts['master']) except SaltClientError: # Failed to update the dns, keep the old addr pass From c92d4100df620ec51189655bd3349d5491678cfa Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 1 Mar 2012 21:06:58 -0800 Subject: [PATCH 169/598] export a few more env vars for apt-get to run non-interactively I tested this by running: salt-call pkg.install postfix Seems to work like a champ and fixes #786 --- salt/modules/apt.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/salt/modules/apt.py b/salt/modules/apt.py index 9a9f01a90963..9829f84fc0a4 100644 --- a/salt/modules/apt.py +++ b/salt/modules/apt.py @@ -2,13 +2,30 @@ Support for APT (Advanced Packaging Tool) ''' +import os def __virtual__(): ''' Confirm this module is on a Debian based system ''' - return 'pkg' if __grains__['os'] in [ 'Debian', 'Ubuntu' ] else False + return 'pkg' if __grains__['os'] in ('Debian', 'Ubuntu') else False + + +def __init__(): + ''' + For Debian and derivative systems, set up + a few env variables to keep apt happy and + non-interactive. + ''' + if __virtual__(): + env_vars = { + 'APT_LISTBUGS_FRONTEND': 'none', + 'APT_LISTCHANGES_FRONTEND': 'none', + 'DEBIAN_FRONTENT': 'noninteractive', + } + # Export these puppies so they persist + os.environ.update(env_vars) def available_version(name): @@ -113,8 +130,7 @@ def install(pkg, refresh=False, repo='', skip_verify=False, ret_pkgs = {} old_pkgs = list_pkgs() - cmd = '{nonint} apt-get -q -y {confold}{verify}{target} install {pkg}'.format( - nonint='DEBIAN_FRONTEND=noninteractive', + cmd = 'apt-get -q -y {confold}{verify}{target} install {pkg}'.format( confold=' -o DPkg::Options::=--force-confold', verify=' --allow-unauthenticated' if skip_verify else '', target=' -t {0}'.format(repo) if repo else '', @@ -150,7 +166,7 @@ def remove(pkg): ret_pkgs = [] old_pkgs = list_pkgs() - cmd = 'DEBIAN_FRONTEND=noninteractive apt-get -q -y remove {0}'.format(pkg) + cmd = 'apt-get -q -y remove {0}'.format(pkg) __salt__['cmd.run'](cmd) new_pkgs = list_pkgs() for pkg in old_pkgs: @@ -175,7 +191,7 @@ def purge(pkg): old_pkgs = list_pkgs() # Remove inital package - purge_cmd = 'DEBIAN_FRONTEND=noninteractive apt-get -q -y purge {0}'.format(pkg) + purge_cmd = 'apt-get -q -y purge {0}'.format(pkg) __salt__['cmd.run'](purge_cmd) new_pkgs = list_pkgs() @@ -211,7 +227,7 @@ def upgrade(refresh=True): ret_pkgs = {} old_pkgs = list_pkgs() - cmd = 'DEBIAN_FRONTEND=noninteractive apt-get -q -y -o DPkg::Options::=--force-confold dist-upgrade' + cmd = 'apt-get -q -y -o DPkg::Options::=--force-confold dist-upgrade' __salt__['cmd.run'](cmd) new_pkgs = list_pkgs() From 7fbcfdb957320b589b1514c9d2d8a95161ee161d Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 1 Mar 2012 21:51:36 -0800 Subject: [PATCH 170/598] Add "upgrade_available" to the apt package module Reference: #698 --- salt/modules/apt.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/salt/modules/apt.py b/salt/modules/apt.py index 9829f84fc0a4..2bd39b9cbfa8 100644 --- a/salt/modules/apt.py +++ b/salt/modules/apt.py @@ -266,3 +266,30 @@ def list_pkgs(regex_string=""): ret[cols[1]] = cols[2] return ret + + +def upgrade_available(name): + + ''' + Check whether or not an upgrade is available for a given package + + CLI Example:: + + salt '*' pkg.upgrade_available <package name> + ''' + cmd = 'apt-get --just-print upgrade' + out = __salt__['cmd.run_stdout'](cmd) + + # Mini filter function + def _to_update(line): + return line.startswith("Conf ") + + upgraded_packages = filter(_to_update, out.split('\n')) + + # Example line: + # 'Conf linux-image-2.6.35-32-generic (2.6.35-32.66 Ubuntu:10.10/maverick-updates [amd64])' + for line in upgraded_packages: + data = line.split() + if name == data[1]: + return True + return False From 03a1c4eda23815e7fd2d232b7964d54961139481 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 1 Mar 2012 22:04:34 -0800 Subject: [PATCH 171/598] Add salt.utils.is_empty() --- salt/utils/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index b0e000eeb214..45c8ab98379e 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -2,9 +2,9 @@ Some of the utils used by salt ''' -import logging import os import sys +import logging log = logging.getLogger(__name__) @@ -43,6 +43,16 @@ '11': 'Nov', '12': 'Dec'} +def is_empty(filename): + ''' + Is a file empty? + ''' + try: + return os.stat(filename).st_size == 0 + except OSError: + # Non-existant file or permission denied to the parent dir + return False + def get_colors(use=True): ''' From ebeb64909082284c681700feaf0227bd0cbc0323 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 1 Mar 2012 22:07:43 -0800 Subject: [PATCH 172/598] Don't try to compile empty sls files to highstate data Fixes #689 --- salt/state.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 7607e46a68ac..e1045c099655 100644 --- a/salt/state.py +++ b/salt/state.py @@ -11,14 +11,15 @@ } ''' +import os import copy import inspect import fnmatch import logging -import os import tempfile import collections +import salt.utils import salt.loader import salt.minion @@ -553,8 +554,12 @@ def compile_template(self, template, env='', sls=''): ''' if not isinstance(template, basestring): return {} + # No highstate data from invalid or empty files if not os.path.isfile(template): return {} + + if salt.utils.is_empty(template): + return {} return self.rend[self.template_shebang(template)](template, env, sls) def compile_template_str(self, template): From da8df9930931fd12657239d7e35aa78c59531b55 Mon Sep 17 00:00:00 2001 From: Tom Prince <tom.prince@ualberta.net> Date: Fri, 2 Mar 2012 01:52:55 -0500 Subject: [PATCH 173/598] Add tests directory to source distribution. --- MANIFEST.in | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7802228b732b..0276299051b5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,10 @@ include AUTHORS include LICENSE include README.rst -include requirements.txt \ No newline at end of file +include requirements.txt +include tests/*.py +recursive-include tests *.py +include tests/integration/modules/files/* +include tests/integration/files/* +include tests/integration/tmp/_README +include tests/unit/templates/files/* \ No newline at end of file From 38551f3f9fd900eda707dbf22600f681b1de8b97 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 1 Mar 2012 23:18:59 -0800 Subject: [PATCH 174/598] Fix another small issue with compiling templates If the file contains only whitespace such as \r, \n, spaces, and tabs, the compile would fail. This checks for that and then does a better job of documenting the checks with comments and stuff. --- salt/state.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index e1045c099655..b27890b5cc96 100644 --- a/salt/state.py +++ b/salt/state.py @@ -552,14 +552,19 @@ def compile_template(self, template, env='', sls=''): Take the path to a template and return the high data structure derived from the template. ''' + # Template was specified incorrectly if not isinstance(template, basestring): return {} - # No highstate data from invalid or empty files + # Template does not exists if not os.path.isfile(template): return {} - + # Template is an empty file if salt.utils.is_empty(template): return {} + # Template is nothing but whitespace + if not open(template).read().strip(): + return {} + return self.rend[self.template_shebang(template)](template, env, sls) def compile_template_str(self, template): From 7170dae5d256f5f71432da0b9e3859aace77f47d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 2 Mar 2012 00:22:40 -0700 Subject: [PATCH 175/598] Make state never return None --- salt/state.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/state.py b/salt/state.py index e1045c099655..e92a04567e27 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1058,6 +1058,8 @@ def render_state(self, sls, env, mods): state[name]['__sls__'] = sls if '__env__' not in state[name]: state[name]['__env__'] = env + else: + state = {} return state, mods, errors def render_highstate(self, matches): From ede77b7d157917a04f1daac791324579faeec4d8 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 2 Mar 2012 11:32:12 -0700 Subject: [PATCH 176/598] add --batch-size to the options for -b --- salt/cli/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 3b83a2ee6961..92c40dc39b79 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -53,6 +53,7 @@ def __parse(self): help='Return the data from minions as the data is returned') parser.add_option('-b', '--batch', + '--batch-size', default='', dest='batch', help=('Execute the salt job in batch mode, pass either the ' From fca44fc9e7beaddb757ca7c24d2434b8918583a4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 2 Mar 2012 12:15:45 -0700 Subject: [PATCH 177/598] fix stack trace with grains error --- salt/loader.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/loader.py b/salt/loader.py index fc57f9803fbb..1c05eb4d50bb 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -496,7 +496,13 @@ def gen_grains(self): for key, fun in funcs.items(): if key[key.index('.') + 1:] == 'core': continue - ret = fun() + try: + ret = fun() + except Exception as exc: + log.critical(('Failed to load grains definded in grain file ' + '{0} in function {1}, error: {2}').format( + key, fun, exc)) + continue if not isinstance(ret, dict): continue grains.update(ret) From 0a88ef6196a65e2e4fdde2f37361ec90b3ddb8f3 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 2 Mar 2012 15:16:36 -0700 Subject: [PATCH 178/598] Add backend for new timeoutless execution --- salt/client.py | 105 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/salt/client.py b/salt/client.py index 7bcfe53269e0..45d961b61f43 100644 --- a/salt/client.py +++ b/salt/client.py @@ -150,6 +150,17 @@ def _check_grain_minions(self, expr): ''' return os.listdir(os.path.join(self.opts['pki_dir'], 'minions')) + def gather_job_info(self, jid, tgt, tgt_type): + ''' + Return the information about a given job + ''' + return self.cmd( + tgt, + 'saltutil.find_job', + [jid], + 2, + tgt_type) + def cmd( self, tgt, @@ -174,6 +185,37 @@ def cmd( timeout=timeout) return self.get_returns(pub_data['jid'], pub_data['minions'], timeout) + def cmd_cli( + self, + tgt, + fun, + arg=(), + timeout=None, + expr_form='glob', + ret=''): + ''' + Execute a salt command and return + ''' + if timeout is None: + timeout = self.opts['timeout'] + jid = prep_jid(self.opts['cachedir']) + pub_data = self.pub( + tgt, + fun, + arg, + expr_form, + ret, + jid=jid, + timeout=timeout) + for fn_ret in self.get_cli_returns(pub_data['jid'], + pub_data['minions'], + timeout, + tgt, + expr_form): + if not fn_ret: + continue + yield fn_ret + def cmd_iter( self, tgt, @@ -252,9 +294,70 @@ def cmd_full_return( ret, jid=jid, timeout=timeout) - return (self.get_full_returns(pub_data['jid'], + return (self.get_returns(pub_data['jid'], pub_data['minions'], timeout)) + def get_cli_returns(self, jid, minions, timeout=None, tgt='*', tgt_type='glob'): + ''' + This method starts off a watcher looking at the return data for + a specified jid, it returns all of the information for the jid + ''' + print 'Executing job with jid {0}'.format(jid) + print '------------------------------------\n' + if timeout is None: + timeout = self.opts['timeout'] + inc_timeout = timeout + jid_dir = os.path.join(self.opts['cachedir'], 'jobs', jid) + start = int(time.time()) + gstart = int(time.time()) + found = set() + wtag = os.path.join(jid_dir, 'wtag*') + # Check to see if the jid is real, if not return the empty dict + if not os.path.isdir(jid_dir): + yield {} + # Wait for the hosts to check in + while True: + for fn_ in os.listdir(jid_dir): + ret = {} + if fn_.startswith('.'): + continue + if fn_ not in found: + retp = os.path.join(jid_dir, fn_, 'return.p') + outp = os.path.join(jid_dir, fn_, 'out.p') + if not os.path.isfile(retp): + continue + while fn_ not in ret: + try: + ret_data = self.serial.load(open(retp, 'r')) + ret[fn_] = {'ret': ret_data} + if os.path.isfile(outp): + ret[fn_]['out'] = self.serial.load(open(outp, 'r')) + except: + pass + found.add(fn_) + yield ret + if glob.glob(wtag) and not int(time.time()) > start + timeout + 1: + # The timeout +1 has not been reached and there is still a + # write tag for the syndic + continue + if len(ret) >= len(minions): + # All minions have returned, break out of the loop + break + if int(time.time()) > start + timeout: + # The timeout has been reached, check the jid to see if the + # timeout needs to be increased + jinfo = self.gather_job_info(jid, tgt, tgt_type) + more_time = False + for id_ in jinfo: + if jinfo[id_]: + print 'Execution is still running on {0}'.format(id_) + more_time = True + if more_time: + timeout += inc_timeout + continue + break + time.sleep(0.01) + def get_iter_returns(self, jid, minions, timeout=None): ''' This method starts off a watcher looking at the return data for From 8cc4f09666272ed2d9c643758c06b3abd4d01135 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 2 Mar 2012 15:17:31 -0700 Subject: [PATCH 179/598] change the default to use the new command structure --- salt/cli/__init__.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 92c40dc39b79..8fbbee52ec47 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -44,13 +44,13 @@ def __parse(self): dest='timeout', help=('Set the return timeout for batch jobs; ' 'default=5 seconds')) - parser.add_option('-i', - '--iter', - '--iter-return', + parser.add_option('-s', + '--static', default=False, - dest='iter_', + dest='static', action='store_true', - help='Return the data from minions as the data is returned') + help=('Return the data from minions as a group after they ' + 'all return.')) parser.add_option('-b', '--batch', '--batch-size', @@ -171,7 +171,7 @@ def __parse(self): if not options.timeout is None: opts['timeout'] = int(options.timeout) - opts['iter'] = options.iter_ + opts['static'] = options.static opts['batch'] = options.batch opts['pcre'] = options.pcre opts['list'] = options.list_ @@ -285,14 +285,14 @@ def run(self): try: # local will be None when there was an error if local: - if self.opts['iter']: - for full_ret in local.cmd_iter(*args): - ret, out = self._format_ret(full_ret) - self._output_ret(ret, out) - else: + if self.opts['static']: full_ret = local.cmd_full_return(*args) ret, out = self._format_ret(full_ret) self._output_ret(ret, out) + else: + for full_ret in local.cmd_cli(*args): + ret, out = self._format_ret(full_ret) + self._output_ret(ret, out) except SaltInvocationError as exc: ret = exc out = '' From edaa807ab095bbef3c3c4b2b14bc1fab4937b867 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 2 Mar 2012 21:28:30 -0700 Subject: [PATCH 180/598] Add verbose option to default command --- salt/client.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/salt/client.py b/salt/client.py index 45d961b61f43..3d41d5eb896b 100644 --- a/salt/client.py +++ b/salt/client.py @@ -192,7 +192,8 @@ def cmd_cli( arg=(), timeout=None, expr_form='glob', - ret=''): + ret='', + verbose=False): ''' Execute a salt command and return ''' @@ -211,7 +212,8 @@ def cmd_cli( pub_data['minions'], timeout, tgt, - expr_form): + expr_form, + verbose): if not fn_ret: continue yield fn_ret @@ -297,13 +299,21 @@ def cmd_full_return( return (self.get_returns(pub_data['jid'], pub_data['minions'], timeout)) - def get_cli_returns(self, jid, minions, timeout=None, tgt='*', tgt_type='glob'): + def get_cli_returns( + self, + jid, + minions, + timeout=None, + tgt='*', + tgt_type='glob', + verbose=False): ''' This method starts off a watcher looking at the return data for a specified jid, it returns all of the information for the jid ''' - print 'Executing job with jid {0}'.format(jid) - print '------------------------------------\n' + if verbose: + print 'Executing job with jid {0}'.format(jid) + print '------------------------------------\n' if timeout is None: timeout = self.opts['timeout'] inc_timeout = timeout @@ -350,7 +360,8 @@ def get_cli_returns(self, jid, minions, timeout=None, tgt='*', tgt_type='glob'): more_time = False for id_ in jinfo: if jinfo[id_]: - print 'Execution is still running on {0}'.format(id_) + if verbose: + print 'Execution is still running on {0}'.format(id_) more_time = True if more_time: timeout += inc_timeout From f84e58962b44cf60479ddf885ac0a9128c97b728 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 2 Mar 2012 21:29:20 -0700 Subject: [PATCH 181/598] Add verbose option to default command --- salt/cli/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 8fbbee52ec47..ac36367fac79 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -51,6 +51,13 @@ def __parse(self): action='store_true', help=('Return the data from minions as a group after they ' 'all return.')) + parser.add_option('-v', + '--verbose', + default=False, + dest='verbose', + action='store_true', + help=('Turn on command verbosity, display jid and active job ' + 'queries')) parser.add_option('-b', '--batch', '--batch-size', @@ -172,6 +179,7 @@ def __parse(self): if not options.timeout is None: opts['timeout'] = int(options.timeout) opts['static'] = options.static + opts['verbose'] = options.verbose opts['batch'] = options.batch opts['pcre'] = options.pcre opts['list'] = options.list_ @@ -282,6 +290,8 @@ def run(self): if self.opts['return']: args.append(self.opts['return']) + else: + args.append('') try: # local will be None when there was an error if local: @@ -290,6 +300,8 @@ def run(self): ret, out = self._format_ret(full_ret) self._output_ret(ret, out) else: + if self.opts['verbose']: + args.append(True) for full_ret in local.cmd_cli(*args): ret, out = self._format_ret(full_ret) self._output_ret(ret, out) From 91bbc10483e00ffc89ed9a769ca53321e26e3383 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 2 Mar 2012 22:24:51 -0700 Subject: [PATCH 182/598] Add provision to fall back to msgpack_pure if msgpack is not installed --- salt/payload.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/payload.py b/salt/payload.py index d47ee630a58e..ad74eb561925 100644 --- a/salt/payload.py +++ b/salt/payload.py @@ -6,7 +6,12 @@ import cPickle as pickle -import msgpack +try: + # Attempt to import msgpack + import msgpack +except ImportError: + # Fall back to msgpack_pure + import msgpack_pure as msgpack def package(payload): From 4981752f1ddac498c74cd97280004aa16aca35cf Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 2 Mar 2012 21:28:26 -0800 Subject: [PATCH 183/598] Fix overlay of command line options in salt-key. Ensure only one definition of default for args which also reside in master config --- salt/cli/__init__.py | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 8fbbee52ec47..5357fb2691aa 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -470,7 +470,7 @@ def __parse(self): parser.add_option('-l', '--list', - dest='list_', + dest='list', default=False, action='store_true', help='List the unaccepted public keys') @@ -526,7 +526,7 @@ def __parse(self): dest='delete', default='', help='Delete the named key') - + parser.add_option('-D', '--delete-all', dest='delete_all', @@ -540,10 +540,9 @@ def __parse(self): default=False, action='store_true', help='Supress output') - + parser.add_option('--key-logfile', dest='key_logfile', - default='/var/log/salt/key.log', help=('Send all output to a file. ' 'Default is /var/log/salt/key.log')) @@ -576,31 +575,19 @@ def __parse(self): options, args = parser.parse_args() opts = {} + opts.update(salt.config.master_config(options.config)) - opts['quiet'] = options.quiet - opts['key_logfile'] = options.key_logfile + for k, v in options.__dict__.items(): + if k == 'keysize': + if v < 2048: + opts[k] = v + else: + opts[k] = v + elif v is not None: + opts[k] = v # I decided to always set this to info, since it really all is info or # error. opts['loglevel'] = 'info' - opts['list'] = options.list_ - opts['list_all'] = options.list_all - opts['accept'] = options.accept - opts['accept_all'] = options.accept_all - opts['reject'] = options.reject - opts['reject_all'] = options.reject_all - opts['print'] = options.print_ - opts['print_all'] = options.print_all - opts['delete'] = options.delete - opts['delete_all'] = options.delete_all - opts['gen_keys'] = options.gen_keys - opts['gen_keys_dir'] = options.gen_keys_dir - if options.keysize < 2048: - opts['keysize'] = 2048 - else: - opts['keysize'] = options.keysize - - opts.update(salt.config.master_config(options.config)) - return opts def run(self): From f0387d5225f49c13adf7132f64155dab06836a8f Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 2 Mar 2012 21:33:20 -0800 Subject: [PATCH 184/598] Allow pidfile to be configured for master in master config in addition to the command line --- salt/__init__.py | 9 ++++----- salt/config.py | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/__init__.py b/salt/__init__.py index 515180276fc0..384b66da7c51 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -96,7 +96,8 @@ def __init__(self): if self.cli['user']: self.opts['user'] = self.cli['user'] # Send the pidfile location to the opts - self.opts['pidfile'] = self.cli['pidfile'] + if self.cli['pidfile']: + self.opts['pidfile'] = self.cli['pidfile'] def __parse_cli(self): ''' @@ -121,9 +122,7 @@ def __parse_cli(self): help='Specify user to run master') parser.add_option('--pid-file', dest='pidfile', - default='/var/run/salt-master.pid', - help=('Specify the location of the pidfile. Default' - ' %default')) + help=('Specify the location of the pidfile.')) parser.add_option('-l', '--log-level', dest='log_level', @@ -171,7 +170,7 @@ def start(self): # Late import so logging works correctly import salt.utils salt.utils.daemonize() - set_pidfile(self.cli['pidfile']) + set_pidfile(self.opts['pidfile']) master.start() diff --git a/salt/config.py b/salt/config.py index e9cd4a39933d..7bb122a853d3 100644 --- a/salt/config.py +++ b/salt/config.py @@ -210,6 +210,7 @@ def master_config(path): 'log_file': '/var/log/salt/master', 'log_level': 'warning', 'log_granular_levels': {}, + 'pidfile': '/var/run/salt-master.pid', 'cluster_masters': [], 'cluster_mode': 'paranoid', 'serial': 'msgpack', From c33f6c5c17f71091f7bb73596a3007f8b0170c81 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 2 Mar 2012 21:53:02 -0800 Subject: [PATCH 185/598] remove unused var --- salt/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/client.py b/salt/client.py index 45d961b61f43..8cc9514a2145 100644 --- a/salt/client.py +++ b/salt/client.py @@ -309,7 +309,6 @@ def get_cli_returns(self, jid, minions, timeout=None, tgt='*', tgt_type='glob'): inc_timeout = timeout jid_dir = os.path.join(self.opts['cachedir'], 'jobs', jid) start = int(time.time()) - gstart = int(time.time()) found = set() wtag = os.path.join(jid_dir, 'wtag*') # Check to see if the jid is real, if not return the empty dict From 222976b691babf6c40d52ded63b9ca55f8d311e4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 2 Mar 2012 23:12:53 -0700 Subject: [PATCH 186/598] Started on #757 Documented some YAML issues we run into when making sls files --- .../troubleshooting/yaml_idiosyncrasies.rst | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 doc/topics/troubleshooting/yaml_idiosyncrasies.rst diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst new file mode 100644 index 000000000000..1d76bd451ae1 --- /dev/null +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -0,0 +1,105 @@ +=================== +YAML Idiosyncrasies +=================== + +One of Salt's strength, the use of existing serialization systems for +representing sls data, can also backfire. YAML is a general purpose system +and there are a number of things that would seem to make sense in an sls +file that cause YAML issues. It is wise to be aware of these issues, while +reports or running into them are generally rare they can still crop up at +unexpected times. + +Spaces vs Tabs +============== + +Yaml uses spaces, period, do not use tabs in your sls files! If strange +errors are coming up in rendering sls files, make sure to check that +no tabs have crept in! + +Indentation +=========== +The suggested +syntax for Yaml files is to use 2 spaces for indentation, but Yaml will +follow whatever indentation system that the individual file uses. Generally +2 space indentation works very well for sls files given the fact that the +represented data is uniform and not deeply nested. + +Nested Dicts (key-value) +------------------------ + +When dicts are more deeply nested they no longer follow the same indentation +logic. This is rarely something that comes up in Salt, since deeply nested +options like these are discouraged when making state modules, but some do +exist. A good example is the context and default options in the file.managed +state: + +.. code-block:: yaml + + /etc/http/conf/http.conf: + file: + - managed + - source: salt://apache/http.conf + - user: root + - group: root + - mode: 644 + - template: jinja + - context: + custom_var: "override" + - defaults: + custom_var: "default value" + other_var: 123 + +Notice that the spacing used is 2 spaces, and that when defining the context +and defaults options there is a 4 space indent. If only a 2 space indent is +used then the information will not be correctly loaded. If using double spacing +is not desireable, then a deeply nested dict can be declared with curly braces: + +.. code-block:: yaml + + /etc/http/conf/http.conf: + file: + - managed + - source: salt://apache/http.conf + - user: root + - group: root + - mode: 644 + - template: jinja + - context: { + custom_var: "override" } + - defaults: { + custom_var: "default value" + other_var: 123 } + +Integers are Parsed as Integers +=============================== + +When passing integers into an sls file, they are passed as integers. This means +that if a state accepts a string value and an integer is passed, that an +integer will be sent. The solution here is to send the integer as a string. + +This is best explained when setting the mode for a file: + +.. code-block:: yaml + + /etc/vimrc: + file: + - managed + - source: salt://edit/vimrc + - user: root + - group: root + - mode: 644 + +Salt manages this well, since the mode is passed as 644, but if the mode is +zero padded as 0644, then it is read by Yaml as an integer and evaluated as +a hexadecimal value, 0644 becomes 420. Therefore, if the file mode is +preceded by a 0 then it needs to be passed as a string: + +.. code-block:: yaml + + /etc/vimrc: + file: + - managed + - source: salt://edit/vimrc + - user: root + - group: root + - mode: '0644' From 9d23c456404164f47e8d2816bb6e6734a423f52a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 2 Mar 2012 23:32:40 -0700 Subject: [PATCH 187/598] Add initial docs for advanced file server settings --- doc/ref/file_server/file_roots.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 doc/ref/file_server/file_roots.rst diff --git a/doc/ref/file_server/file_roots.rst b/doc/ref/file_server/file_roots.rst new file mode 100644 index 000000000000..0f71c50b133c --- /dev/null +++ b/doc/ref/file_server/file_roots.rst @@ -0,0 +1,26 @@ +========================= +File Server Configuration +========================= + +The Salt file server is a high performance file server written in ZeroMQ. It +manages large files quickly and with little overhead, and has been optomized +to handle small files in an extreamly efficient manner. + +The Salt file server is an environment aware file server, this means that +files can be allocated within many root directories and accessed by +specifying both the file path and the environment to search. The +individual environments can also be spanned across mutiple directory roots +to crate overlays and to allow for files to be orginaized in many flexible +ways. + +Environments +============ + +The Salt file server defaults to the mandatory ``base`` environment. This +environment MUST be defined and is used to download files when no +environment is specified. + +Environments allow for files and sls data to be logically seperated, but +environments are not isolated from each other. This allows for logical +isolation of environments by the engineer using Salt, but also allows +for information to be used in multiple environments for maximum flexibility. From faf32055941ba42135d0232bf91486a654c848a9 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 2 Mar 2012 23:06:02 -0800 Subject: [PATCH 188/598] More cli overlay refator, normalize the use of conf_file for config file path --- salt/cli/__init__.py | 61 ++++++++++++++---------------------------- salt/runners/jobs.py | 4 +-- salt/runners/manage.py | 4 +-- 3 files changed, 24 insertions(+), 45 deletions(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 5357fb2691aa..7e6671d305dc 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -169,23 +169,12 @@ def __parse(self): opts = {} + for k, v in options.__dict__.items(): + if v is not None: + opts[k] = v + if not options.timeout is None: opts['timeout'] = int(options.timeout) - opts['static'] = options.static - opts['batch'] = options.batch - opts['pcre'] = options.pcre - opts['list'] = options.list_ - opts['grain'] = options.grain - opts['grain_pcre'] = options.grain_pcre - opts['exsel'] = options.exsel - opts['nodegroup'] = options.nodegroup - opts['compound'] = options.compound - opts['return'] = options.return_ - opts['conf_file'] = options.conf_file - opts['raw_out'] = options.raw_out - opts['txt_out'] = options.txt_out - opts['yaml_out'] = options.yaml_out - opts['json_out'] = options.json_out if options.query: opts['query'] = options.query @@ -426,13 +415,9 @@ def __parse(self): opts = {} - opts['timeout'] = options.timeout - opts['pcre'] = options.pcre - opts['list'] = options.list_ - opts['grain'] = options.grain - opts['grain_pcre'] = options.grain_pcre - opts['nodegroup'] = options.nodegroup - opts['conf_file'] = options.conf_file + for k, v in options.__dict__.items(): + if v is not None: + opts[k] = v if len(args) <= 1: parser.print_help() @@ -568,14 +553,14 @@ def __parse(self): parser.add_option('-c', '--config', - dest='config', + dest='conf_file', default='/etc/salt/master', help='Pass in an alternative configuration file') options, args = parser.parse_args() opts = {} - opts.update(salt.config.master_config(options.config)) + opts.update(salt.config.master_config(options.conf_file)) for k, v in options.__dict__.items(): if k == 'keysize': @@ -628,7 +613,7 @@ def __parse(self): 'from, multiple directories can be delimited by commas')) parser.add_option('-c', '--config', - dest='config', + dest='conf_file', default='/etc/salt/minion', help='Pass in an alternative configuration file') parser.add_option('-d', @@ -675,17 +660,14 @@ def __parse(self): options, args = parser.parse_args() opts = {} + opts.update(salt.config.minion_config(options.conf_file)) + + for k, v in options.__dict__.items(): + if k == 'module_dirs': + opts[k] = v.split(',') + else: + opts[k] = v - opts['grains_run'] = options.grains - opts['module_dirs'] = options.module_dirs.split(',') - opts['doc'] = options.doc - opts['raw_out'] = options.raw_out - opts['txt_out'] = options.txt_out - opts['yaml_out'] = options.yaml_out - opts['color'] = not options.no_color - opts['json_out'] = options.json_out - opts.update(salt.config.minion_config(options.config)) - opts['log_level'] = options.log_level if len(args) >= 1: opts['fun'] = args[0] opts['arg'] = args[1:] @@ -727,7 +709,7 @@ def __parse(self): parser.add_option('-c', '--config', - dest='config', + dest='conf_file', default='/etc/salt/master', help=('Change the location of the master configuration; ' 'default=/etc/salt/master')) @@ -745,10 +727,9 @@ def __parse(self): options, args = parser.parse_args() opts = {} - - opts['config'] = options.config + opts.update(salt.config.master_config(options.conf_file)) + opts['conf_file'] = options.conf_file opts['doc'] = options.doc - if len(args) > 0: opts['fun'] = args[0] else: @@ -758,8 +739,6 @@ def __parse(self): else: opts['arg'] = [] - opts.update(salt.config.master_config(options.config)) - return opts def run(self): diff --git a/salt/runners/jobs.py b/salt/runners/jobs.py index 74ec9c21493c..0f0cff778825 100644 --- a/salt/runners/jobs.py +++ b/salt/runners/jobs.py @@ -20,7 +20,7 @@ def active(): ''' ret = {} job_dir = os.path.join(__opts__['cachedir'], 'jobs') - client = salt.client.LocalClient(__opts__['config']) + client = salt.client.LocalClient(__opts__['conf_file']) active = client.cmd('*', 'saltutil.running', timeout=1) for minion, data in active.items(): if not isinstance(data, tuple): @@ -67,7 +67,7 @@ def _format_ret(full_ret): out = data['out'] return ret, out - client = salt.client.LocalClient(__opts__['config']) + client = salt.client.LocalClient(__opts__['conf_file']) full_ret = client.get_full_returns(jid, [], 0) ret, out = _format_ret(full_ret) # Determine the proper output method and run it diff --git a/salt/runners/manage.py b/salt/runners/manage.py index 0dc15237dbeb..969d77c53cb4 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -11,7 +11,7 @@ def down(): ''' Print a list of all the down or unresponsive salt minions ''' - client = salt.client.LocalClient(__opts__['config']) + client = salt.client.LocalClient(__opts__['conf_file']) key = salt.cli.key.Key(__opts__) minions = client.cmd('*', 'test.ping', timeout=1) keys = key._keys('acc') @@ -27,7 +27,7 @@ def up(): ''' Print a list of all of the minions that are up ''' - client = salt.client.LocalClient(__opts__['config']) + client = salt.client.LocalClient(__opts__['conf_file']) minions = client.cmd('*', 'test.ping', timeout=1) for minion in sorted(minions): From 0ce6db7e33c67e998ee72869526ebed64ea5c1d4 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 2 Mar 2012 23:12:28 -0800 Subject: [PATCH 189/598] Use context manager so file is guaranteed to close after block --- salt/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/__init__.py b/salt/__init__.py index 384b66da7c51..ff57fc50c5a1 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -28,7 +28,8 @@ def set_pidfile(pidfile): if not os.path.isdir(pdir): os.makedirs(pdir) try: - open(pidfile, 'w+').write(str(os.getpid())) + with open(pidfile, 'w+') as f: + f.write(str(os.getpid())) except IOError: pass From 26dfcafd66d7f3f646abbd72cd70f9a1d22ababd Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 2 Mar 2012 23:22:21 -0800 Subject: [PATCH 190/598] Use tuple for checking multiple suffixes with endswith --- salt/loader.py | 7 ++----- salt/state.py | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/salt/loader.py b/salt/loader.py index 1c05eb4d50bb..ff0e2d25519c 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -337,11 +337,8 @@ def gen_functions(self, pack=None, virtual_enable=True): for fn_ in os.listdir(mod_dir): if fn_.startswith('_'): continue - if fn_.endswith('.py')\ - or fn_.endswith('.pyc')\ - or fn_.endswith('.pyo')\ - or fn_.endswith('.so')\ - or (cython_enabled and fn_.endswith('.pyx')): + if (fn_.endswith(('.py', '.pyc', '.pyo', '.so')) + or (cython_enabled and fn_.endswith('.pyx'))): names[fn_[:fn_.rindex('.')]] = os.path.join(mod_dir, fn_) for name in names: try: diff --git a/salt/state.py b/salt/state.py index 8d991537533a..4db99955cbde 100644 --- a/salt/state.py +++ b/salt/state.py @@ -162,11 +162,8 @@ def module_refresh(self, data): ''' if data['state'] == 'file': if data['fun'] == 'managed': - if any((data['name'].endswith('.py'), - data['name'].endswith('.pyx'), - data['name'].endswith('.pyo'), - data['name'].endswith('.pyc'), - data['name'].endswith('.so'))): + if data['name'].endswith( + ('.py', '.pyx', '.pyo', '.pyc', '.so')): self.load_modules() open(os.path.join( self.opts['cachedir'], From b0abefc1577c23fdb1ddf6b41e51932300124db3 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 2 Mar 2012 23:55:40 -0800 Subject: [PATCH 191/598] fix bad dest var names --- salt/cli/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 7e6671d305dc..ac6b1a5c8732 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -69,7 +69,7 @@ def __parse(self): parser.add_option('-L', '--list', default=False, - dest='list_', + dest='list', action='store_true', help=('Instead of using shell globs to evaluate the target ' 'servers, take a comma delimited list of servers.')) @@ -119,7 +119,7 @@ def __parse(self): 'webser* or E@database.*\'')) parser.add_option('--return', default='', - dest='return_', + dest='return', metavar='RETURNER', help=('Set an alternative return method. By default salt will ' 'send the return data from the command back to the ' @@ -128,7 +128,6 @@ def __parse(self): parser.add_option('-Q', '--query', dest='query', - default=False, action='store_true', help=('This option is deprecated and will be removed in a ' 'future release, please use salt-run jobs instead\n' @@ -374,7 +373,7 @@ def __parse(self): parser.add_option('-L', '--list', default=False, - dest='list_', + dest='list', action='store_true', help=('Instead of using shell globs to evaluate the target ' 'servers, take a comma delimited list of servers.')) From f0d05c13d95e3552fa59bb7160a5bf6db72f78d9 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 2 Mar 2012 23:56:29 -0800 Subject: [PATCH 192/598] Refactor common loader code --- salt/loader.py | 56 +++++++++++++++----------------------------------- 1 file changed, 17 insertions(+), 39 deletions(-) diff --git a/salt/loader.py b/salt/loader.py index ff0e2d25519c..c7d89021bb12 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -17,20 +17,25 @@ salt_base_path = os.path.dirname(salt.__file__) -def minion_mods(opts): - ''' - Returns the minion modules - ''' +def _create_loader(opts, ext_type, tag): extra_dirs = [ os.path.join(opts['extension_modules'], - 'modules') + ext_type) ] - if 'module_dirs' in opts: - extra_dirs.extend(opts['module_dirs']) + ext_type_dirs = '%s_dirs' % ext_type + if ext_type_dirs in opts: + extra_dirs.extend(opts[ext_type_dirs]) module_dirs = [ - os.path.join(salt_base_path, 'modules'), + os.path.join(salt_base_path, ext_type), ] + extra_dirs - load = Loader(module_dirs, opts, 'module') + return Loader(module_dirs, opts, tag) + + +def minion_mods(opts): + ''' + Returns the minion modules + ''' + load = _create_loader(opts, 'modules', 'module') return load.apply_introspection(load.gen_functions()) @@ -38,16 +43,7 @@ def raw_mod(opts, name, functions): ''' Returns a single module loaded raw and bypassing the __virtual__ function ''' - extra_dirs = [ - os.path.join(opts['extension_modules'], - 'modules') - ] - if 'module_dirs' in opts: - extra_dirs.extend(opts['module_dirs']) - module_dirs = [ - os.path.join(salt_base_path, 'modules'), - ] + extra_dirs - load = Loader(module_dirs, opts, 'rawmodule') + load = _create_loader(opts, 'modules', 'rawmodule') return load.gen_module(name, functions) @@ -55,16 +51,7 @@ def returners(opts): ''' Returns the returner modules ''' - extra_dirs = [ - os.path.join(opts['extension_modules'], - 'returners') - ] - if 'returner_dirs' in opts: - extra_dirs.extend(opts['returner_dirs']) - module_dirs = [ - os.path.join(salt_base_path, 'returners'), - ] + extra_dirs - load = Loader(module_dirs, opts, 'returner') + load = _create_loader(opts, 'returners', 'returner') return load.filter_func('returner') @@ -72,16 +59,7 @@ def states(opts, functions): ''' Returns the returner modules ''' - extra_dirs = [ - os.path.join(opts['extension_modules'], - 'states') - ] - if 'states_dirs' in opts: - extra_dirs.extend(opts['states_dirs']) - module_dirs = [ - os.path.join(salt_base_path, 'states'), - ] + extra_dirs - load = Loader(module_dirs, opts, 'state') + load = _create_loader(opts, 'states', 'state') pack = {'name': '__salt__', 'value': functions} return load.gen_functions(pack) From 05970e4e6e5fe546c583ef985bf3b054a2da70e1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 3 Mar 2012 08:27:08 -0700 Subject: [PATCH 193/598] Track full returns within the return retrival function for the cli --- salt/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/client.py b/salt/client.py index dbf98b8356ad..83f908bf9452 100644 --- a/salt/client.py +++ b/salt/client.py @@ -316,6 +316,7 @@ def get_cli_returns( print '------------------------------------\n' if timeout is None: timeout = self.opts['timeout'] + fret = {} inc_timeout = timeout jid_dir = os.path.join(self.opts['cachedir'], 'jobs', jid) start = int(time.time()) @@ -344,12 +345,13 @@ def get_cli_returns( except: pass found.add(fn_) + fret.update(ret) yield ret if glob.glob(wtag) and not int(time.time()) > start + timeout + 1: # The timeout +1 has not been reached and there is still a # write tag for the syndic continue - if len(ret) >= len(minions): + if len(fret) >= len(minions): # All minions have returned, break out of the loop break if int(time.time()) > start + timeout: From 6810ee21cddd988108dba62ef00dc4e09fa58281 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 3 Mar 2012 09:19:16 -0800 Subject: [PATCH 194/598] Add extension_modules to root path, Issue #758 --- salt/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/config.py b/salt/config.py index 7bb122a853d3..f64815ab8c2f 100644 --- a/salt/config.py +++ b/salt/config.py @@ -175,7 +175,8 @@ def minion_config(path): opts['grains'] = salt.loader.grains(opts) # Prepend root_dir to other paths - prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', 'key_logfile']) + prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', + 'key_logfile', 'extension_modules']) return opts From 51e1227aab4264e11ffb6acbdfd7d57277b898fa Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 3 Mar 2012 09:33:36 -0800 Subject: [PATCH 195/598] When jid is 0, returnerers are empty, Issue #822 --- salt/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/client.py b/salt/client.py index 8cc9514a2145..945d6614d288 100644 --- a/salt/client.py +++ b/salt/client.py @@ -422,6 +422,9 @@ def get_returns(self, jid, minions, timeout=None): gstart = int(time.time()) ret = {} wtag = os.path.join(jid_dir, 'wtag*') + # If jid == 0, there is no payload + if int(jid) == 0: + return ret # Check to see if the jid is real, if not return the empty dict if not os.path.isdir(jid_dir): return ret From bb1a005ece95579014e23990fadba71a1d6c2fdf Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 3 Mar 2012 18:08:02 -0800 Subject: [PATCH 196/598] Avoid duplicating grain loading in salt-call, skip verify, correct dest in cli parser The salt-call command was loading the grains twice, once in the minion configuration and once at command run. Both loads are the same. In addition, the parser was verifying the pki, logdir and cache dir, but this is not important since everything is run locally from the extension_modules path and printed to stdout. --- salt/cli/__init__.py | 8 ++------ salt/cli/caller.py | 4 ++-- salt/config.py | 1 + 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index ac6b1a5c8732..9c2fe6742485 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -600,7 +600,7 @@ def __parse(self): parser.add_option('-g', '--grains', - dest='grains', + dest='grains_run', default=False, action='store_true', help='Return the information generated by the salt grains') @@ -673,10 +673,6 @@ def __parse(self): else: opts['fun'] = '' opts['arg'] = [] - salt.verify_env([opts['pki_dir'], - opts['cachedir'], - os.path.dirname(opts['log_file']), - ]) return opts @@ -742,7 +738,7 @@ def __parse(self): def run(self): ''' - Execute the salt call! + Execute salt-run ''' runner = salt.runner.Runner(self.opts) runner.run() diff --git a/salt/cli/caller.py b/salt/cli/caller.py index e80345786043..e792cc8cbfa6 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -15,6 +15,7 @@ # Custom exceptions from salt.exceptions import CommandExecutionError, CommandNotFoundError + class Caller(object): ''' Object to wrap the calling of local salt modules for the salt-call command @@ -24,7 +25,6 @@ def __init__(self, opts): Pass in the command line options ''' self.opts = opts - opts['grains'] = salt.loader.grains(opts) self.minion = salt.minion.SMinion(opts) def call(self): @@ -113,4 +113,4 @@ def run(self): if 'json_out' in self.opts and self.opts['json_out']: printout.indent = 2 - printout({'local': ret['return']}, color=self.opts['color']) + printout({'local': ret['return']}, color=self.opts['no_color']) diff --git a/salt/config.py b/salt/config.py index f64815ab8c2f..794dc179fcfe 100644 --- a/salt/config.py +++ b/salt/config.py @@ -152,6 +152,7 @@ def minion_config(path): 'state_verbose': False, 'acceptance_wait_time': 10, 'dns_check': True, + 'grains': {}, } load_config(opts, path, 'SALT_MINION_CONFIG') From 109985d5a3e0672c4875d71cc11ad85de3c5e6f3 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 3 Mar 2012 18:21:50 -0800 Subject: [PATCH 197/598] Add verification of env back into salt-call --- salt/cli/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 9c2fe6742485..a14238fb672d 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -674,6 +674,11 @@ def __parse(self): opts['fun'] = '' opts['arg'] = [] + salt.verify_env([opts['pki_dir'], + opts['cachedir'], + os.path.dirname(opts['log_file']), + ]) + return opts def run(self): From e25037f08f4806c1d245bcfb8a8d54310c8832a6 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 3 Mar 2012 18:29:20 -0800 Subject: [PATCH 198/598] use set notation instead of iteration --- salt/runners/manage.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/salt/runners/manage.py b/salt/runners/manage.py index 969d77c53cb4..3ab34d028a65 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -16,10 +16,7 @@ def down(): minions = client.cmd('*', 'test.ping', timeout=1) keys = key._keys('acc') - for minion in minions: - keys.remove(minion) - - for minion in sorted(keys): + for minion in sorted(keys - set(minions.keys())): print minion From ba491efb9926f0dba0c64b608a4b9076e5f4052d Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 3 Mar 2012 18:35:20 -0800 Subject: [PATCH 199/598] Fix another bad dest var name --- salt/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index a14238fb672d..daedf147e623 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -494,7 +494,7 @@ def __parse(self): parser.add_option('-p', '--print', - dest='print_', + dest='print', default='', help='Print the specified public key') From 881fa2a7d31dc2a523a2fb45e79b6522eae464cb Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 3 Mar 2012 18:40:50 -0800 Subject: [PATCH 200/598] Fix issue with salt-run jobs.lookup_jid on unfinished jobs Fixes #824 --- salt/runners/jobs.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/salt/runners/jobs.py b/salt/runners/jobs.py index 0f0cff778825..7ce378470b40 100644 --- a/salt/runners/jobs.py +++ b/salt/runners/jobs.py @@ -9,6 +9,7 @@ import salt.client import salt.payload import salt.utils +from salt.exceptions import SaltException # Import Third party libs import yaml @@ -55,12 +56,13 @@ def lookup_jid(jid): Return the printout from a previousely executed job ''' + out = None + def _format_ret(full_ret): ''' Take the full return data and format it to simple output ''' ret = {} - out = '' for key, data in full_ret.items(): ret[key] = data['ret'] if 'out' in data: @@ -69,17 +71,24 @@ def _format_ret(full_ret): client = salt.client.LocalClient(__opts__['conf_file']) full_ret = client.get_full_returns(jid, [], 0) - ret, out = _format_ret(full_ret) + formatted = _format_ret(full_ret) + + if formatted: + ret = formatted[0] + out = formatted[1] + else: + ret = SaltException('Job {0} hasn\'t finished. No data yet :('.format(jid)) + out = '' + # Determine the proper output method and run it get_outputter = salt.output.get_outputter - if isinstance(ret, list) or isinstance(ret, dict): - if out: - printout = get_outputter(out) - else: - printout = get_outputter(None) + if isinstance(ret, (list, dict, basestring)) and out: + printout = get_outputter(out) # Pretty print any salt exceptions elif isinstance(ret, SaltException): printout = get_outputter("txt") + else: + printout = get_outputter(None) printout(ret) return ret From 3c9e0dfd9cd160b15ceea087193fd3fe37df1233 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 3 Mar 2012 23:20:23 -0700 Subject: [PATCH 201/598] initial commit of new fileclient module This module enables file clients to follow a unified design and be made to interact with any available file cache, the basic idea being that the file client interface does not need to be the master per-se, but can be a local file_roots, or another file server that suppiles similar interfaces as the salt file server. --- salt/fileclient.py | 572 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 572 insertions(+) create mode 100644 salt/fileclient.py diff --git a/salt/fileclient.py b/salt/fileclient.py new file mode 100644 index 000000000000..e447a727a257 --- /dev/null +++ b/salt/fileclient.py @@ -0,0 +1,572 @@ +''' +Classes that manage file clients +''' +# Import python libs +import BaseHTTPServer +import contextlib +import logging +import hashlib +import os +import shutil +import string +import subprocess +import urllib2 +import urlparse + +# Import third-party libs +import yaml +import zmq + +# Import salt libs +from salt.exceptions import MinionError +import salt.client +import salt.crypt +import salt.loader +import salt.utils +import salt.payload + +log = logging.getLogger(__name__) + +class Client(object): + ''' + Base class for Salt file interactions + ''' + def __init__(self, opts): + self.opts = opts + self.serial = salt.payload.Serial(self.opts) + + def _check_proto(self, path): + ''' + Make sure that this path is intended for the salt master and trim it + ''' + if not path.startswith('salt://'): + raise MinionError('Unsupported path: {0}'.format(path)) + return path[7:] + + def _file_local_list(self, dest): + ''' + Helper util to return a list of files in a directory + ''' + if os.path.isdir(dest): + destdir = dest + else: + destdir = os.path.dirname(dest) + + filelist = [] + + for root, dirs, files in os.walk(destdir): + for name in files: + path = os.path.join(root, name) + filelist.append(path) + + return filelist + + def _cache_loc(self, path, env='base'): + ''' + Return the local location to cache the file, cache dirs will be made + ''' + dest = os.path.join( + self.opts['cachedir'], + 'files', + env, + path + ) + destdir = os.path.dirname(dest) + cumask = os.umask(191) + if not os.path.isdir(destdir): + os.makedirs(destdir) + os.chmod(dest, 384) + os.umask(cumask) + return dest + + def cache_file(self, path, env='base'): + ''' + Pull a file down from the file server and store it in the minion + file cache + ''' + return self.get_url(path, '', True, env) + + def cache_files(self, paths, env='base'): + ''' + Download a list of files stored on the master and put them in the + minion file cache + ''' + ret = [] + for path in paths: + ret.append(self.cache_file(path, env)) + return ret + + def cache_master(self, env='base'): + ''' + Download and cache all files on a master in a specified environment + ''' + ret = [] + for path in self.file_list(env): + ret.append(self.cache_file('salt://{0}'.format(path), env)) + return ret + + def cache_dir(self, path, env='base'): + ''' + Download all of the files in a subdir of the master + ''' + ret = [] + path = self._check_proto(path) + for fn_ in self.file_list(env): + if fn_.startswith(path): + local = self.cache_file('salt://{0}'.format(fn_), env) + if not fn_.strip(): + continue + ret.append(local) + return ret + + def cache_local_file(self, path, **kwargs): + ''' + Cache a local file on the minion in the localfiles cache + ''' + dest = os.path.join(self.opts['cachedir'], 'localfiles', + path.lstrip('/')) + destdir = os.path.dirname(dest) + + if not os.path.isdir(destdir): + os.makedirs(destdir) + + shutil.copyfile(path, dest) + return dest + + def file_local_list(self, env='base'): + ''' + List files in the local minion files and localfiles caches + ''' + filesdest = os.path.join(self.opts['cachedir'], 'files', env) + localfilesdest = os.path.join(self.opts['cachedir'], 'localfiles') + + return sorted(self._file_local_list(filesdest) + + self._file_local_list(localfilesdest)) + + def file_list(self, env='base'): + ''' + This function must be overwritten + ''' + return [] + + def is_cached(self, path, env='base'): + ''' + Returns the full path to a file if it is cached locally on the minion + otherwise returns a blank string + ''' + localsfilesdest = os.path.join( + self.opts['cachedir'], 'localfiles', path.lstrip('/')) + filesdest = os.path.join( + self.opts['cachedir'], 'files', env, path.lstrip('salt://')) + + if os.path.exists(filesdest): + return filesdest + elif os.path.exists(localsfilesdest): + return localsfilesdest + + return '' + + def get_state(self, sls, env): + ''' + Get a state file from the master and store it in the local minion + cache return the location of the file + ''' + if '.' in sls: + sls = sls.replace('.', '/') + for path in ['salt://' + sls + '.sls', + os.path.join('salt://', sls, 'init.sls')]: + dest = self.cache_file(path, env) + if dest: + return dest + return False + + def get_dir(self, path, dest='', env='base'): + ''' + Get a directory recursively from the salt-master + ''' + # TODO: We need to get rid of using the string lib in here + ret = [] + # Strip trailing slash + path = string.rstrip(self._check_proto(path), '/') + # Break up the path into a list containing the bottom-level directory + # (the one being recursively copied) and the directories preceding it + separated = string.rsplit(path,'/',1) + if len(separated) != 2: + # No slashes in path. (This means all files in env will be copied) + prefix = '' + else: + prefix = separated[0] + + # Copy files from master + for fn_ in self.file_list(env): + if fn_.startswith(path): + # Remove the leading directories from path to derive + # the relative path on the minion. + minion_relpath = string.lstrip(fn_[len(prefix):],'/') + ret.append(self.get_file('salt://{0}'.format(fn_), + '%s/%s' % (dest,minion_relpath), + True, env)) + # Replicate empty dirs from master + for fn_ in self.file_list_emptydirs(env): + if fn_.startswith(path): + # Remove the leading directories from path to derive + # the relative path on the minion. + minion_relpath = string.lstrip(fn_[len(prefix):],'/') + minion_mkdir = '%s/%s' % (dest,minion_relpath) + os.makedirs(minion_mkdir) + ret.append(minion_mkdir) + ret.sort() + return ret + + def get_url(self, url, dest, makedirs=False, env='base'): + ''' + Get a single file from a URL. + ''' + url_data = urlparse.urlparse(url) + if url_data.scheme == 'salt': + return self.get_file(url, dest, makedirs, env) + if dest: + destdir = os.path.dirname(dest) + if not os.path.isdir(destdir): + if makedirs: + os.makedirs(destdir) + else: + return '' + else: + dest = os.path.join( + self.opts['cachedir'], + 'extrn_files', + env, + os.path.join( + url_data.netloc, + os.path.relpath(url_data.path, '/')) + ) + destdir = os.path.dirname(dest) + if not os.path.isdir(destdir): + os.makedirs(destdir) + try: + with contextlib.closing(urllib2.urlopen(url)) as srcfp: + with open(dest, 'wb') as destfp: + shutil.copyfileobj(srcfp, destfp) + return dest + except urllib2.HTTPError, ex: + raise MinionError('HTTP error {0} reading {1}: {3}'.format( + ex.code, + url, + *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) + except urllib2.URLError, ex: + raise MinionError('Error reading {0}: {1}'.format(url, ex.reason)) + return '' + +class LocalClient(Client): + ''' + Use the local_roots option to parse a local file root + ''' + def __init__(self, opts): + Client.__init__(self, opts) + + def _find_file(self, path, env='base'): + ''' + Locate the file path + ''' + fnd = {'path': '', + 'rel': ''} + if env not in self.opts['file_roots']: + return fnd + for root in self.opts['file_roots'][env]: + full = os.path.join(root, path) + if os.path.isfile(full): + fnd['path'] = full + fnd['rel'] = path + return fnd + return fnd + + def get_file(self, path, dest='', makedirs=False, env='base'): + ''' + Coppies a file from the local files directory and coppies it into place + ''' + path = self._check_proto(path) + fnd = self._find_file(path, env) + if not dest: + dest = _cache_loc(path, env) + destdir = os.path.dirname(dest) + if not os.path.isdir(destdir): + if makedirs: + os.makedirs(destdir) + else: + return False + shutil.copy(fnd['path'], dest) + return dest + + def file_list(self, env='base'): + ''' + Return a list of files in the given environment + ''' + ret = [] + if env not in self.opts['file_roots']: + return ret + for path in self.opts['file_roots'][env]: + for root, dirs, files in os.walk(path): + for fn in files: + ret.append( + os.path.relpath( + os.path.join( + root, + fn + ), + path + ) + ) + return ret + + def file_list_emptydirs(self, env='base'): + ''' + List the empty dirs in the file_roots + ''' + ret = [] + if env not in self.opts['file_roots']: + return ret + for path in self.opts['file_roots'][env]: + for root, dirs, files in os.walk(path): + if len(dirs)==0 and len(files)==0: + ret.append(os.path.relpath(root,path)) + return ret + + def hash_file(self, path, env='base'): + ''' + Return the hash of a file, to get the hash of a file in the file_roots + prepend the path with salt://<file on server> otherwise, prepend the + file with / for a local file. + ''' + ret = {} + try: + path = self._check_proto(path) + except MinionError: + if not os.path.isfile(path): + err = ('Specified file {0} is not present to generate ' + 'hash').format(path) + log.warning(err) + return ret + else: + ret['hsum'] = hashlib.md5(open(path, 'rb').read()).hexdigest() + ret['hash_type'] = 'md5' + return ret + path = self._find_file(path, env)['path'] + if not path: + return {} + ret = {} + ret['hsum'] = getattr(hashlib, self.opts['hash_type'])( + open(path, 'rb').read()).hexdigest() + ret['hash_type'] = self.opts['hash_type'] + return ret + + def list_env(self, path, env='base'): + ''' + Return a list of the files in the file server's specified environment + ''' + return self.file_list(env) + + def master_opts(self): + ''' + Return the master opts data + ''' + return self.opts + + def ext_nodes(self): + ''' + Return the metadata derived from the external nodes system on the local + system + ''' + if not self.opts['external_nodes']: + return {} + if not salt.utils.which(self.opts['external_nodes']): + log.error(('Specified external nodes controller {0} is not' + ' available, please verify that it is installed' + '').format(self.opts['external_nodes'])) + return {} + cmd = '{0} {1}'.format(self.opts['external_nodes'], self.opts['id']) + ndata = yaml.safe_load( + subprocess.Popen( + cmd, + shell=True, + stdout=subprocess.PIPE + ).communicate()[0]) + ret = {} + if 'environment' in ndata: + env = ndata['environment'] + else: + env = 'base' + + if 'classes' in ndata: + if isinstance(ndata['classes'], dict): + ret[env] = ndata['classes'].keys() + elif isinstance(ndata['classes'], list): + ret[env] = ndata['classes'] + else: + return ret + return ret + + +class RemoteClient(Client): + ''' + Interact with the salt master file server. + ''' + def __init__(self, opts): + Client.__init__(self, opts) + self.auth = salt.crypt.SAuth(opts) + self.socket = self.__get_socket() + + def __get_socket(self): + ''' + Return the ZeroMQ socket to use + ''' + context = zmq.Context() + socket = context.socket(zmq.REQ) + socket.connect(self.opts['master_uri']) + return socket + + def get_file(self, path, dest='', makedirs=False, env='base'): + ''' + Get a single file from the salt-master + path must be a salt server location, aka, salt://path/to/file, if + dest is ommited, then the downloaded file will be placed in the minion + cache + ''' + path = self._check_proto(path) + payload = {'enc': 'aes'} + fn_ = None + if dest: + destdir = os.path.dirname(dest) + if not os.path.isdir(destdir): + if makedirs: + os.makedirs(destdir) + else: + return False + fn_ = open(dest, 'w+') + load = {'path': path, + 'env': env, + 'cmd': '_serve_file'} + while True: + if not fn_: + load['loc'] = 0 + else: + load['loc'] = fn_.tell() + payload['load'] = self.auth.crypticle.dumps(load) + self.socket.send(self.serial.dumps(payload)) + data = self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) + if not data['data']: + if not fn_ and data['dest']: + # This is a 0 byte file on the master + dest = os.path.join( + self.opts['cachedir'], + 'files', + env, + data['dest'] + ) + destdir = os.path.dirname(dest) + cumask = os.umask(191) + if not os.path.isdir(destdir): + os.makedirs(destdir) + if not os.path.exists(dest): + open(dest, 'w+').write(data['data']) + os.chmod(dest, 384) + os.umask(cumask) + break + if not fn_: + dest = os.path.join( + self.opts['cachedir'], + 'files', + env, + data['dest'] + ) + destdir = os.path.dirname(dest) + cumask = os.umask(191) + if not os.path.isdir(destdir): + os.makedirs(destdir) + fn_ = open(dest, 'w+') + os.chmod(dest, 384) + os.umask(cumask) + fn_.write(data['data']) + return dest + + def file_list(self, env='base'): + ''' + List the files on the master + ''' + payload = {'enc': 'aes'} + load = {'env': env, + 'cmd': '_file_list'} + payload['load'] = self.auth.crypticle.dumps(load) + self.socket.send(self.serial.dumps(payload)) + return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) + + def file_list_emptydirs(self, env='base'): + ''' + List the empty dirs on the master + ''' + payload = {'enc': 'aes'} + load = {'env': env, + 'cmd': '_file_list_emptydirs'} + payload['load'] = self.auth.crypticle.dumps(load) + self.socket.send(self.serial.dumps(payload)) + return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) + + def hash_file(self, path, env='base'): + ''' + Return the hash of a file, to get the hash of a file on the salt + master file server prepend the path with salt://<file on server> + otherwise, prepend the file with / for a local file. + ''' + try: + path = self._check_proto(path) + except MinionError: + if not os.path.isfile(path): + err = ('Specified file {0} is not present to generate ' + 'hash').format(path) + log.warning(err) + return {} + else: + ret = {} + ret['hsum'] = hashlib.md5(open(path, 'rb').read()).hexdigest() + ret['hash_type'] = 'md5' + return ret + payload = {'enc': 'aes'} + load = {'path': path, + 'env': env, + 'cmd': '_file_hash'} + payload['load'] = self.auth.crypticle.dumps(load) + self.socket.send(self.serial.dumps(payload)) + return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) + + def list_env(self, path, env='base'): + ''' + Return a list of the files in the file server's specified environment + ''' + payload = {'enc': 'aes'} + load = {'env': env, + 'cmd': '_file_list'} + payload['load'] = self.auth.crypticle.dumps(load) + self.socket.send(self.serial.dumps(payload)) + return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) + + def master_opts(self): + ''' + Return the master opts data + ''' + payload = {'enc': 'aes'} + load = {'cmd': '_master_opts'} + payload['load'] = self.auth.crypticle.dumps(load) + self.socket.send(self.serial.dumps(payload)) + return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) + + def ext_nodes(self): + ''' + Return the metadata derived from the external nodes system on the + master. + ''' + payload = {'enc': 'aes'} + load = {'cmd': '_ext_nodes', + 'id': self.opts['id']} + payload['load'] = self.auth.crypticle.dumps(load) + self.socket.send(self.serial.dumps(payload)) + return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) From 6aeb31ae9bb0a11d885efe68f189bc950c9007af Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 3 Mar 2012 20:12:26 -0800 Subject: [PATCH 202/598] Use with statments for file handling when possible, drive by pep8 --- salt/utils/jinja.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 7bd939ec1421..5ec840746167 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -4,6 +4,7 @@ from jinja2.exceptions import TemplateNotFound import salt + def get_template(filename, opts, env): loader = SaltCacheLoader(opts, env) if filename.startswith(loader.searchpath): @@ -14,7 +15,9 @@ def get_template(filename, opts, env): return jinja.get_template(relpath) else: # fallback for templates outside the state tree - return Template(open(filename, 'r').read()) + with open(filename, 'r') as f: + return Template(f.read()) + class SaltCacheLoader(BaseLoader): ''' @@ -31,7 +34,7 @@ def __init__(self, opts, env='base', encoding='utf-8'): self.searchpath = path.join(opts['cachedir'], 'files', env) self._file_client = None self.cached = [] - + def file_client(self): ''' Return a file client. Instantiates on first call. @@ -60,14 +63,13 @@ def get_source(self, environment, template): template = path.join(*split_template_path(template)) self.check_cache(template) filepath = path.join(self.searchpath, template) - try: - f = open(filepath, 'rb') - contents = f.read().decode(self.encoding) - except IOError: - raise TemplateNotFound(template) - finally: - f.close() + with open(filepath, 'rb') as f: + try: + contents = f.read().decode(self.encoding) + except IOError: + raise TemplateNotFound(template) mtime = path.getmtime(filepath) + def uptodate(): try: return path.getmtime(filepath) == mtime From b53cca532a64a1a03d7f6c62dba2b155fa3ec925 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 3 Mar 2012 21:58:20 -0800 Subject: [PATCH 203/598] One set of parens is enough --- salt/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/config.py b/salt/config.py index 794dc179fcfe..e5b3c494cf76 100644 --- a/salt/config.py +++ b/salt/config.py @@ -33,8 +33,8 @@ def _validate_file_roots(file_roots): just replace it with an empty list ''' if not isinstance(file_roots, dict): - log.warning(('The file_roots parameter is not properly formatted,' - ' using defaults')) + log.warning('The file_roots parameter is not properly formatted,' + ' using defaults') return {'base': ['/srv/salt']} for env, dirs in file_roots.items(): if not isinstance(dirs, list) and not isinstance(dirs, tuple): From 929ac0718a89f2ff843f1d57fcf384a3c0c20bd6 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 3 Mar 2012 22:07:35 -0800 Subject: [PATCH 204/598] Use template string replacement over raw concatenation --- salt/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/config.py b/salt/config.py index e5b3c494cf76..63d22134b0cc 100644 --- a/salt/config.py +++ b/salt/config.py @@ -162,8 +162,8 @@ def minion_config(path): opts['master_ip'] = dns_check(opts['master']) - opts['master_uri'] = 'tcp://' + opts['master_ip'] + ':'\ - + str(opts['master_port']) + opts['master_uri'] = 'tcp://%s:%s' % (opts['master_ip'], + opts['master_port']) # Enabling open mode requires that the value be set to True, and # nothing else! From d1e6f5caa80c0b66ad71012383fc868ec8fc2e89 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 3 Mar 2012 22:18:16 -0800 Subject: [PATCH 205/598] Refactor common yaml file loading of configuration --- salt/config.py | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/salt/config.py b/salt/config.py index 63d22134b0cc..bbea0a235c72 100644 --- a/salt/config.py +++ b/salt/config.py @@ -3,6 +3,7 @@ ''' # Import python modules +from contextlib import nested import glob import os import sys @@ -42,6 +43,15 @@ def _validate_file_roots(file_roots): return file_roots +def _read_conf_file(path): + with open(path, 'r') as conf_file: + conf_opts = yaml.safe_load(conf_file.read()) or {} + # allow using numeric ids: convert int to string + if 'id' in conf_opts: + conf_opts['id'] = str(conf_opts['id']) + return conf_opts + + def load_config(opts, path, env_var): ''' Attempts to update ``opts`` dict by parsing either the file described by @@ -54,22 +64,13 @@ def load_config(opts, path, env_var): if not os.path.isfile(path): template = '{0}.template'.format(path) if os.path.isfile(template): - with open(path, 'w') as out: - with open(template, 'r') as f: - f.readline() # skip first line - out.write(f.read()) + with nested(open(path, 'w'), open(template, 'r')) as (out, f): + f.readline() # skip first line + out.write(f.read()) if os.path.isfile(path): try: - conf_opts = yaml.safe_load(open(path, 'r')) - if conf_opts is None: - # The config file is empty and the yaml.load returned None - conf_opts = {} - else: - # allow using numeric ids: convert int to string - if 'id' in conf_opts: - conf_opts['id'] = str(conf_opts['id']) - opts.update(conf_opts) + opts.update(_read_conf_file(path)) opts['conf_file'] = path except Exception, e: msg = 'Error parsing configuration file: {0} - {1}' @@ -91,15 +92,7 @@ def include_config(opts, orig_path): path = os.path.join(os.path.dirname(orig_path), path) for fn_ in glob.glob(path): try: - conf_opts = yaml.safe_load(open(fn_, 'r')) - if conf_opts is None: - # The config file is empty and the yaml.load returned None - conf_opts = {} - else: - # allow using numeric ids: convert int to string - if 'id' in conf_opts: - conf_opts['id'] = str(conf_opts['id']) - opts.update(conf_opts) + opts.update(_read_conf_file(path)) except Exception, e: msg = 'Error parsing configuration file: {0} - {1}' log.warn(msg.format(fn_, e)) From ea2ef21cd569044251d94fcb9c51622b7541dddb Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 3 Mar 2012 22:20:08 -0800 Subject: [PATCH 206/598] Do not check for 'grains' key in opts, it will always have a grains key --- salt/loader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/loader.py b/salt/loader.py index c7d89021bb12..a75fb5726108 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -120,8 +120,7 @@ def grains(opts): opts['grains'] = {} load = Loader(module_dirs, opts, 'grain') grains = load.gen_grains() - if 'grains' in opts: - grains.update(opts['grains']) + grains.update(opts['grains']) return grains From 0c232954d373c25c9fc6a133ec5ad4c33993d93c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 3 Mar 2012 23:51:05 -0700 Subject: [PATCH 207/598] Add get_file_client to return uniform file client based on opts --- salt/fileclient.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/fileclient.py b/salt/fileclient.py index e447a727a257..202a9a1243db 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -27,6 +27,19 @@ log = logging.getLogger(__name__) +def get_file_client(opts): + ''' + Read in the ``file_server`` option and return the correct type of file + server + ''' + try: + return { + 'remote': RemoteClient, + 'local': LocalClient + }.get(opts['file_server'], 'remote')(opts) + except KeyError: + return RemoteClient(opts) + class Client(object): ''' Base class for Salt file interactions From f31ecd781f124e6df35eb56ac59b5b354dd5784f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 4 Mar 2012 00:09:03 -0700 Subject: [PATCH 208/598] remove FileClient from minion.py --- salt/minion.py | 361 ------------------------------------------------- 1 file changed, 361 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 426f7d50fd73..3e33b31c40be 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -702,364 +702,3 @@ def nodegroup_match(self, tgt, nodegroups): if tgt in nodegroups: return self.compound_match(nodegroups[tgt]) return False - - -class FileClient(object): - ''' - Interact with the salt master file server. - ''' - def __init__(self, opts): - self.opts = opts - self.serial = salt.payload.Serial(self.opts) - self.auth = salt.crypt.SAuth(opts) - self.socket = self.__get_socket() - - def __get_socket(self): - ''' - Return the ZeroMQ socket to use - ''' - context = zmq.Context() - socket = context.socket(zmq.REQ) - socket.connect(self.opts['master_uri']) - return socket - - def _check_proto(self, path): - ''' - Make sure that this path is intended for the salt master and trim it - ''' - if not path.startswith('salt://'): - raise MinionError('Unsupported path: {0}'.format(path)) - return path[7:] - - def _file_local_list(self, dest): - ''' - Helper util to return a list of files in a directory - ''' - if os.path.isdir(dest): - destdir = dest - else: - destdir = os.path.dirname(dest) - - filelist = [] - - for root, dirs, files in os.walk(destdir): - for name in files: - path = os.path.join(root, name) - filelist.append(path) - - return filelist - - def get_file(self, path, dest='', makedirs=False, env='base'): - ''' - Get a single file from the salt-master - ''' - path = self._check_proto(path) - payload = {'enc': 'aes'} - fn_ = None - if dest: - destdir = os.path.dirname(dest) - if not os.path.isdir(destdir): - if makedirs: - os.makedirs(destdir) - else: - return False - fn_ = open(dest, 'w+') - load = {'path': path, - 'env': env, - 'cmd': '_serve_file'} - while True: - if not fn_: - load['loc'] = 0 - else: - load['loc'] = fn_.tell() - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - data = self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - if not data['data']: - if not fn_ and data['dest']: - # This is a 0 byte file on the master - dest = os.path.join( - self.opts['cachedir'], - 'files', - env, - data['dest'] - ) - destdir = os.path.dirname(dest) - cumask = os.umask(191) - if not os.path.isdir(destdir): - os.makedirs(destdir) - if not os.path.exists(dest): - open(dest, 'w+').write(data['data']) - os.chmod(dest, 384) - os.umask(cumask) - break - if not fn_: - dest = os.path.join( - self.opts['cachedir'], - 'files', - env, - data['dest'] - ) - destdir = os.path.dirname(dest) - cumask = os.umask(191) - if not os.path.isdir(destdir): - os.makedirs(destdir) - fn_ = open(dest, 'w+') - os.chmod(dest, 384) - os.umask(cumask) - fn_.write(data['data']) - return dest - - def get_dir(self, path, dest='', env='base'): - ''' - Get a directory recursively from the salt-master - ''' - ret = [] - # Strip trailing slash - path = string.rstrip(self._check_proto(path), '/') - # Break up the path into a list containing the bottom-level directory - # (the one being recursively copied) and the directories preceding it - separated = string.rsplit(path,'/',1) - if len(separated) != 2: - # No slashes in path. (This means all files in env will be copied) - prefix = '' - else: - prefix = separated[0] - - # Copy files from master - for fn_ in self.file_list(env): - if fn_.startswith(path): - # Remove the leading directories from path to derive - # the relative path on the minion. - minion_relpath = string.lstrip(fn_[len(prefix):],'/') - ret.append(self.get_file('salt://{0}'.format(fn_), - '%s/%s' % (dest,minion_relpath), - True, env)) - # Replicate empty dirs from master - for fn_ in self.file_list_emptydirs(env): - if fn_.startswith(path): - # Remove the leading directories from path to derive - # the relative path on the minion. - minion_relpath = string.lstrip(fn_[len(prefix):],'/') - minion_mkdir = '%s/%s' % (dest,minion_relpath) - os.makedirs(minion_mkdir) - ret.append(minion_mkdir) - ret.sort() - return ret - - def get_url(self, url, dest, makedirs=False, env='base'): - ''' - Get a single file from a URL. - ''' - url_data = urlparse.urlparse(url) - if url_data.scheme == 'salt': - return self.get_file(url, dest, makedirs, env) - if dest: - destdir = os.path.dirname(dest) - if not os.path.isdir(destdir): - if makedirs: - os.makedirs(destdir) - else: - return '' - else: - dest = os.path.join( - self.opts['cachedir'], - 'extrn_files', - env, - os.path.join( - url_data.netloc, - os.path.relpath(url_data.path, '/')) - ) - destdir = os.path.dirname(dest) - if not os.path.isdir(destdir): - os.makedirs(destdir) - try: - with contextlib.closing(urllib2.urlopen(url)) as srcfp: - with open(dest, 'wb') as destfp: - shutil.copyfileobj(srcfp, destfp) - return dest - except urllib2.HTTPError, ex: - raise MinionError('HTTP error {0} reading {1}: {3}'.format( - ex.code, - url, - *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) - except urllib2.URLError, ex: - raise MinionError('Error reading {0}: {1}'.format(url, ex.reason)) - return '' - - def cache_file(self, path, env='base'): - ''' - Pull a file down from the file server and store it in the minion - file cache - ''' - return self.get_url(path, '', True, env) - - def cache_files(self, paths, env='base'): - ''' - Download a list of files stored on the master and put them in the - minion file cache - ''' - ret = [] - for path in paths: - ret.append(self.cache_file(path, env)) - return ret - - def cache_master(self, env='base'): - ''' - Download and cache all files on a master in a specified environment - ''' - ret = [] - for path in self.file_list(env): - ret.append(self.cache_file('salt://{0}'.format(path), env)) - return ret - - def cache_dir(self, path, env='base'): - ''' - Download all of the files in a subdir of the master - ''' - ret = [] - path = self._check_proto(path) - for fn_ in self.file_list(env): - if fn_.startswith(path): - local = self.cache_file('salt://{0}'.format(fn_), env) - if not fn_.strip(): - continue - ret.append(local) - return ret - - def cache_local_file(self, path, **kwargs): - ''' - Cache a local file on the minion in the localfiles cache - ''' - dest = os.path.join(self.opts['cachedir'], 'localfiles', - path.lstrip('/')) - destdir = os.path.dirname(dest) - - if not os.path.isdir(destdir): - os.makedirs(destdir) - - shutil.copyfile(path, dest) - return dest - - def file_list(self, env='base'): - ''' - List the files on the master - ''' - payload = {'enc': 'aes'} - load = {'env': env, - 'cmd': '_file_list'} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - - def file_list_emptydirs(self, env='base'): - ''' - List the empty dirs on the master - ''' - payload = {'enc': 'aes'} - load = {'env': env, - 'cmd': '_file_list_emptydirs'} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - - def file_local_list(self, env='base'): - ''' - List files in the local minion files and localfiles caches - ''' - filesdest = os.path.join(self.opts['cachedir'], 'files', env) - localfilesdest = os.path.join(self.opts['cachedir'], 'localfiles') - - return sorted(self._file_local_list(filesdest) + - self._file_local_list(localfilesdest)) - - def is_cached(self, path, env='base'): - ''' - Returns the full path to a file if it is cached locally on the minion - otherwise returns a blank string - ''' - localsfilesdest = os.path.join( - self.opts['cachedir'], 'localfiles', path.lstrip('/')) - filesdest = os.path.join( - self.opts['cachedir'], 'files', env, path.lstrip('salt://')) - - if os.path.exists(filesdest): - return filesdest - elif os.path.exists(localsfilesdest): - return localsfilesdest - - return '' - - def hash_file(self, path, env='base'): - ''' - Return the hash of a file, to get the hash of a file on the salt - master file server prepend the path with salt://<file on server> - otherwise, prepend the file with / for a local file. - ''' - try: - path = self._check_proto(path) - except MinionError: - if not os.path.isfile(path): - err = ('Specified file {0} is not present to generate ' - 'hash').format(path) - log.warning(err) - return {} - else: - ret = {} - ret['hsum'] = hashlib.md5(open(path, 'rb').read()).hexdigest() - ret['hash_type'] = 'md5' - return ret - payload = {'enc': 'aes'} - load = {'path': path, - 'env': env, - 'cmd': '_file_hash'} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - - def list_env(self, path, env='base'): - ''' - Return a list of the files in the file server's specified environment - ''' - payload = {'enc': 'aes'} - load = {'env': env, - 'cmd': '_file_list'} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - - def get_state(self, sls, env): - ''' - Get a state file from the master and store it in the local minion - cache return the location of the file - ''' - if '.' in sls: - sls = sls.replace('.', '/') - for path in ['salt://' + sls + '.sls', - os.path.join('salt://', sls, 'init.sls')]: - dest = self.cache_file(path, env) - if dest: - return dest - return False - - def master_opts(self): - ''' - Return the master opts data - ''' - payload = {'enc': 'aes'} - load = {'cmd': '_master_opts'} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - - def ext_nodes(self): - ''' - Return the metadata derived from the external nodes system on the - master. - ''' - payload = {'enc': 'aes'} - load = {'cmd': '_ext_nodes', - 'id': self.opts['id']} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) From 519b1ff2515841e6a1c5df48855295a0c0a28094 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 4 Mar 2012 00:09:34 -0700 Subject: [PATCH 209/598] Change state to use new fileclient interface --- salt/state.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 4db99955cbde..1fea73e59997 100644 --- a/salt/state.py +++ b/salt/state.py @@ -11,6 +11,7 @@ } ''' +# Import python libs import os import copy import inspect @@ -19,9 +20,11 @@ import tempfile import collections +# Import Salt Libs import salt.utils import salt.loader import salt.minion +import salt.fileclient log = logging.getLogger(__name__) @@ -797,7 +800,7 @@ class HighState(object): salt master or in the local cache. ''' def __init__(self, opts): - self.client = salt.minion.FileClient(opts) + self.client = salt.fileclient.get_file_client(opts) self.opts = self.__gen_opts(opts) self.state = State(self.opts) self.matcher = salt.minion.Matcher(self.opts) From a4dedc2f0a56788ad263d4d3f5bb05bd93a96860 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 4 Mar 2012 00:10:10 -0700 Subject: [PATCH 210/598] change cp module to use fileclient interface --- salt/modules/cp.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/salt/modules/cp.py b/salt/modules/cp.py index 355c9e1f7712..cdc0ad0f851e 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -1,10 +1,12 @@ ''' Minion side functions for salt-cp ''' - +# Import python libs import os +# Import salt libs import salt.minion +import salt.fileclient def recv(files, dest): @@ -46,7 +48,7 @@ def get_file(path, dest, env='base'): if not hash_file(path,env): return '' else: - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.get_file(path, dest, False, env) @@ -58,7 +60,7 @@ def get_dir(path, dest, env='base'): salt '*' cp.get_dir salt://path/to/dir/ /minion/dest ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.get_dir(path, dest, env) @@ -71,7 +73,7 @@ def get_url(path, dest, env='base'): salt '*' cp.get_url salt://my/file /tmp/mine salt '*' cp.get_url http://www.slashdot.org /tmp/index.html ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.get_url(path, dest, False, env) @@ -83,7 +85,7 @@ def cache_file(path, env='base'): salt '*' cp.cache_file salt://path/to/file ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.cache_file(path, env) @@ -97,7 +99,7 @@ def cache_files(paths, env='base'): salt '*' cp.cache_files salt://pathto/file1,salt://pathto/file1 ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.cache_files(paths, env) @@ -109,7 +111,7 @@ def cache_dir(path, env='base'): salt '*' cp.cache_dir salt://path/to/dir ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.cache_dir(path, env) @@ -121,7 +123,7 @@ def cache_master(env='base'): salt '*' cp.cache_master ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.cache_master(env) @@ -147,7 +149,7 @@ def cache_local_file(path): return path_cached # The file hasn't been cached or has changed; cache it - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.cache_local_file(path) @@ -159,7 +161,7 @@ def list_master(env='base'): salt '*' cp.list_master ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.file_list(env) @@ -171,7 +173,7 @@ def list_minion(env='base'): salt '*' cp.list_minion ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.file_local_list(env) @@ -184,7 +186,7 @@ def is_cached(path, env='base'): salt '*' cp.is_cached salt://path/to/file ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.is_cached(path, env) @@ -198,5 +200,5 @@ def hash_file(path, env='base'): salt '*' cp.hash_file salt://path/to/file ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.hash_file(path, env) From 7516374da7836c6ac4d90e279278b30c59c9ac75 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 4 Mar 2012 00:11:24 -0700 Subject: [PATCH 211/598] update jinja utils module to use new fileclient interface --- salt/utils/jinja.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 7bd939ec1421..68cd33927ac6 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -1,8 +1,17 @@ +''' +Jinja loading utils to enable a more powerful backend for jinja templates +''' +# Import python libs from os import path + +# Import third-party libs from jinja2 import Template, BaseLoader, Environment from jinja2.loaders import split_template_path from jinja2.exceptions import TemplateNotFound + +# Import Salt libs import salt +import salt.fileclient def get_template(filename, opts, env): loader = SaltCacheLoader(opts, env) @@ -37,7 +46,7 @@ def file_client(self): Return a file client. Instantiates on first call. ''' if not self._file_client: - self._file_client = salt.minion.FileClient(self.opts) + self._file_client = salt.fileclient.get_file_client(self.opts) return self._file_client def cache_file(self, template): From 4fa502bc598cbb8d732ce255c1cbdb0853a6a52b Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sun, 4 Mar 2012 09:26:42 -0800 Subject: [PATCH 212/598] Use u+rwx for cachedir makedirs. This allows the use of salt when run as a non-root user on a minion. --- salt/minion.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 426f7d50fd73..2b042b89c7cb 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -12,6 +12,7 @@ import os import re import shutil +import stat import string import socket import tempfile @@ -140,7 +141,6 @@ def __load_modules(self): Return the functions and the returners loaded up from the loader module ''' - self.opts['grains'] = salt.loader.grains(self.opts) functions = salt.loader.minion_mods(self.opts) returners = salt.loader.returners(self.opts) return functions, returners @@ -785,12 +785,12 @@ def get_file(self, path, dest='', makedirs=False, env='base'): data['dest'] ) destdir = os.path.dirname(dest) - cumask = os.umask(191) + cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO) if not os.path.isdir(destdir): os.makedirs(destdir) if not os.path.exists(dest): open(dest, 'w+').write(data['data']) - os.chmod(dest, 384) + os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) os.umask(cumask) break if not fn_: @@ -801,11 +801,11 @@ def get_file(self, path, dest='', makedirs=False, env='base'): data['dest'] ) destdir = os.path.dirname(dest) - cumask = os.umask(191) + cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO) if not os.path.isdir(destdir): os.makedirs(destdir) fn_ = open(dest, 'w+') - os.chmod(dest, 384) + os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) os.umask(cumask) fn_.write(data['data']) return dest From 1bdd2bad6bcbfd3e90e34ce1b24444a4a471f00c Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sun, 4 Mar 2012 09:28:43 -0800 Subject: [PATCH 213/598] Use with statements when working with files in states.file --- salt/states/file.py | 59 +++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 97065158fe86..a7c79c8c6948 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -82,6 +82,7 @@ ''' import codecs +from contextlib import nested # For < 2.7 compat import os import shutil import difflib @@ -208,8 +209,9 @@ def _mako(sfn, name, source, user, group, mode, env, context=None): passthrough = context if context else {} passthrough.update(__salt__) passthrough.update(__grains__) - template = Template(open(sfn, 'r').read()) - open(tgt, 'w+').write(template.render(**passthrough)) + with nested(open(sfn, 'r'), open(tgt, 'w+')) as (src, target): + template = Template(src.read()) + target.write(template.render(**passthrough)) return {'result': True, 'data': tgt} except: @@ -234,8 +236,9 @@ def _jinja(sfn, name, source, user, group, mode, env, context=None): 'data': 'Failed to import jinja'} try: newline = False - if open(sfn, 'rb').read().endswith('\n'): - newline = True + with open(sfn, 'rb') as source: + if source.read().endswith('\n'): + newline = True tgt = tempfile.mkstemp()[1] passthrough = context if context else {} passthrough['salt'] = __salt__ @@ -248,11 +251,15 @@ def _jinja(sfn, name, source, user, group, mode, env, context=None): passthrough['env'] = env template = get_template(sfn, __opts__, env) try: - open(tgt, 'w+').write(template.render(**passthrough)) + with open(tgt, 'w+') as target: + target.write(template.render(**passthrough)) + if newline: + target.write('\n') except UnicodeEncodeError: - codecs.open(tgt, encoding='utf-8', mode='w+').write(template.render(**passthrough)) - if newline: - open(tgt, 'a').write('\n') + with codecs.open(tgt, encoding='utf-8', mode='w+') as target: + target.write(template.render(**passthrough)) + if newline: + target.write('\n') return {'result': True, 'data': tgt} except: @@ -289,7 +296,8 @@ def _py(sfn, name, source, user, group, mode, env, context=None): try: tgt = tempfile.mkstemp()[1] - open(tgt, 'w+').write(mod.run()) + with open(tgt, 'w+') as target: + target.write(mod.run()) return {'result': True, 'data': tgt} except: @@ -543,7 +551,9 @@ def managed(name, return ret if data['result']: sfn = data['data'] - hsum = hashlib.md5(open(sfn, 'r').read()).hexdigest() + hsum = '' + with open(sfn, 'r') as source: + hsum = hashlib.md5(source.read()).hexdigest() source_sum = {'hash_type': 'md5', 'hsum': hsum} else: @@ -571,7 +581,9 @@ def managed(name, source_hash ) return ret - comps = open(hash_fn, 'r').read().split('=') + comps = [] + with open(hash_fn, 'r') as hashfile: + comps = hashfile.read().split('=') if len(comps) < 2: ret['result'] = False ret['comment'] = ('Source hash file {0} contains an ' @@ -650,8 +662,10 @@ def managed(name, # Only test the checksums on files with managed contents if source: - name_sum = getattr(hashlib, source_sum['hash_type'])(open(name, - 'rb').read()).hexdigest() + name_sum = '' + hash_func = getattr(hashlib, source_sum['hash_type']) + with open(name, 'rb') as namefile: + name_sum = hash_func(namefile.read()).hexdigest() # Check if file needs to be replaced if source and source_sum['hsum'] != name_sum: @@ -665,12 +679,13 @@ def managed(name, if _is_bin(sfn) or _is_bin(name): ret['changes']['diff'] = 'Replace binary file' else: - slines = open(sfn, 'rb').readlines() - nlines = open(name, 'rb').readlines() + with nested(open(sfn, 'rb'), open(name, 'rb')) as (src, name_): + slines = src.readlines() + nlines = name_.readlines() # Print a diff equivalent to diff -u old new - ret['changes']['diff'] = (''.join(difflib - .unified_diff(nlines, - slines))) + ret['changes']['diff'] = (''.join(difflib + .unified_diff(nlines, + slines))) # Pre requisites are met, and the file needs to be replaced, do it if not __opts__['test']: shutil.copyfile(sfn, name) @@ -717,7 +732,6 @@ def managed(name, # Create the file, user-rw-only if mode will be set if mode: cumask = os.umask(384) - open(name, 'a+').close() if mode: os.umask(cumask) ret['changes']['new'] = 'file {0} created'.format(name) @@ -1049,9 +1063,12 @@ def recurse(name, _makedirs(dest) if os.path.isfile(dest): keep.add(dest) + srch = '' + dsth = '' # The file is present, if the sum differes replace it - srch = hashlib.md5(open(fn_, 'r').read()).hexdigest() - dsth = hashlib.md5(open(dest, 'r').read()).hexdigest() + with nested(open(fn_, 'r'), open(dest, 'r')) as (src_, dst_): + srch = hashlib.md5(src_.read()).hexdigest() + dsth = hashlib.md5(dst_.read()).hexdigest() if srch != dsth: # The downloaded file differes, replace! # FIXME: no metadata (ownership, permissions) available From 56441ddda4e90658f7629eea24a12bea8dc94adf Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sun, 4 Mar 2012 10:40:54 -0800 Subject: [PATCH 214/598] Quote state which caused error in messages for clarity --- salt/state.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/state.py b/salt/state.py index 1fea73e59997..0c41d59d06e2 100644 --- a/salt/state.py +++ b/salt/state.py @@ -295,7 +295,7 @@ def verify_high(self, high): if state.startswith('__'): continue if not isinstance(body[state], list): - err = ('The state {0} in sls {1} is not formed as a list' + err = ('The state "{0}" in sls {1} is not formed as a list' .format(name, body['__sls__'])) errors.append(err) else: @@ -310,7 +310,7 @@ def verify_high(self, high): or arg.keys()[0] == 'watch': if not isinstance(arg[arg.keys()[0]], list): errors.append(('The require or watch' - ' statement in state {0} in sls {1} needs' + ' statement in state "{0}" in sls "{1}" needs' ' to be formed as a list').format( name, body['__sls__'] @@ -329,17 +329,17 @@ def verify_high(self, high): # Make sure that there is only one key in the dict if len(arg.keys()) != 1: errors.append(('Multiple dictionaries defined' - ' in argument of state {0} in sls {1}').format( + ' in argument of state "{0}" in sls {1}').format( name, body['__sls__'])) if not fun: if state == 'require' or state == 'watch': continue - errors.append(('No function declared in state {0} in' + errors.append(('No function declared in state "{0}" in' ' sls {1}').format(state, body['__sls__'])) elif fun > 1: errors.append(('Too many functions declared in state' - ' {0} in sls {1}').format(state, body['__sls__'])) + ' "{0}" in sls {1}').format(state, body['__sls__'])) return errors def verify_chunks(self, chunks): From d458954e7282bf42658a8e0f66f5f818248102cb Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sun, 4 Mar 2012 10:45:53 -0800 Subject: [PATCH 215/598] Use str.format for template string --- salt/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/config.py b/salt/config.py index bbea0a235c72..9cf1b5e2a2d7 100644 --- a/salt/config.py +++ b/salt/config.py @@ -155,8 +155,8 @@ def minion_config(path): opts['master_ip'] = dns_check(opts['master']) - opts['master_uri'] = 'tcp://%s:%s' % (opts['master_ip'], - opts['master_port']) + opts['master_uri'] = 'tcp://{ip}:{port}'.format(ip=opts['master_ip'], + port=opts['master_port']) # Enabling open mode requires that the value be set to True, and # nothing else! From 15accc5becd205e7baad03e43b61d43549fbf993 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sun, 4 Mar 2012 11:07:35 -0800 Subject: [PATCH 216/598] Fix dynamic reloading of grains --- salt/minion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/minion.py b/salt/minion.py index b74196bcd4df..3bd16e3dd107 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -141,6 +141,7 @@ def __load_modules(self): Return the functions and the returners loaded up from the loader module ''' + self.opts['grains'] = salt.loader.grains(self.opts) functions = salt.loader.minion_mods(self.opts) returners = salt.loader.returners(self.opts) return functions, returners From 17c91adae7bb9c7bf10cd7405fd73e11411be86a Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sun, 4 Mar 2012 11:25:40 -0800 Subject: [PATCH 217/598] Refactor common code in state.module_refresh --- salt/state.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/salt/state.py b/salt/state.py index 0c41d59d06e2..24c27eda1556 100644 --- a/salt/state.py +++ b/salt/state.py @@ -163,28 +163,23 @@ def module_refresh(self, data): python, pyx, or .so. Always refresh if the function is recuse, since that can lay down anything. ''' + def _refresh(): + self.load_modules() + module_refresh_path = os.path.join( + self.opts['cachedir'], + 'module_refresh') + with open(module_refresh_path, 'w+') as f: + f.write('') + if data['state'] == 'file': if data['fun'] == 'managed': if data['name'].endswith( ('.py', '.pyx', '.pyo', '.pyc', '.so')): - self.load_modules() - open(os.path.join( - self.opts['cachedir'], - 'module_refresh'), - 'w+').write('') + _refresh() elif data['fun'] == 'recurse': - self.load_modules() - open(os.path.join( - self.opts['cachedir'], - 'module_refresh'), - 'w+').write('') + _refresh() elif data['state'] == 'pkg': - self.load_modules() - open(os.path.join( - self.opts['cachedir'], - 'module_refresh'), - 'w+').write('') - + _refresh() def format_verbosity(self, returns): ''' From 0c907dfc7a4a30944a90d478ef8caf728e46d81a Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sun, 4 Mar 2012 11:35:30 -0800 Subject: [PATCH 218/598] Use a property to recalculate the master_pub uri. This should allow dns updates for the master name to be used when reconnecting --- salt/minion.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 3bd16e3dd107..efe46e85421b 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -398,6 +398,11 @@ def _return_pub(self, ret, ret_cmd='_return'): open(fn_, 'w+').write(self.serial.dumps(ret)) return ret_val + @property + def master_pub(self): + return 'tcp://{ip}:{port}'.format(ip=self.opts['master_ip'], + port=self.publish_port) + def authenticate(self): ''' Authenticate with the master, this method breaks the functional @@ -432,14 +437,10 @@ def tune_in(self): ''' Lock onto the publisher. This is the main event loop for the minion ''' - master_pub = 'tcp://{0}:{1}'.format( - self.opts['master_ip'], - str(self.publish_port) - ) context = zmq.Context() socket = context.socket(zmq.SUB) socket.setsockopt(zmq.SUBSCRIBE, '') - socket.connect(master_pub) + socket.connect(self.master_pub) if self.opts['sub_timeout']: last = time.time() while True: @@ -464,7 +465,7 @@ def tune_in(self): socket.close() socket = context.socket(zmq.SUB) socket.setsockopt(zmq.SUBSCRIBE, '') - socket.connect(master_pub) + socket.connect(self.master_pub) last = time.time() time.sleep(0.05) multiprocessing.active_children() From 1752715a9eabce503a1385cbe66629bf3a68c258 Mon Sep 17 00:00:00 2001 From: blast_hardcheese <blast@hardchee.se> Date: Sun, 4 Mar 2012 18:42:36 -0800 Subject: [PATCH 219/598] Reordering directories in _create_loader The goal of this commit (not sure if I achieved it) was for user-specified _modules to override builtins. --- salt/loader.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/salt/loader.py b/salt/loader.py index a75fb5726108..399378d0efdd 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -18,16 +18,22 @@ def _create_loader(opts, ext_type, tag): - extra_dirs = [ - os.path.join(opts['extension_modules'], - ext_type) - ] + '''Creates Loader instance + + Order of module_dirs: + opts[ext_type_dirs], + extension types, + base types. + ''' + ext_types = os.path.join(opts['extension_modules'], ext_type) + sys_types = os.path.join(salt_base_path, ext_type) + + ext_type_types = [] ext_type_dirs = '%s_dirs' % ext_type if ext_type_dirs in opts: - extra_dirs.extend(opts[ext_type_dirs]) - module_dirs = [ - os.path.join(salt_base_path, ext_type), - ] + extra_dirs + ext_type_types.extend(opts[ext_type_dirs]) + + module_dirs = ext_type_types + [ext_types, sys_types] return Loader(module_dirs, opts, tag) From c03da4206ff6de9b0a548b64af7afcde9731356b Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 4 Mar 2012 21:49:20 -0700 Subject: [PATCH 220/598] Add local file client opts to minion --- salt/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/config.py b/salt/config.py index 9cf1b5e2a2d7..0a11df3543e9 100644 --- a/salt/config.py +++ b/salt/config.py @@ -127,6 +127,11 @@ def minion_config(path): 'failhard': False, 'autoload_dynamic_modules': True, 'environment': None, + 'file_roots': { + 'base': ['/srv/salt'], + }, + 'hash_type': 'md5', + 'external_nodes': '', 'disable_modules': [], 'disable_returners': [], 'module_dirs': [], From fd792c1caa5b3b6fc2556bdea40a33d3ce7303a7 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 4 Mar 2012 21:59:39 -0700 Subject: [PATCH 221/598] Add file directory settings to the minion config template --- conf/minion.template | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/conf/minion.template b/conf/minion.template index 2ae3bae13d0d..eb2a6b09f73a 100644 --- a/conf/minion.template +++ b/conf/minion.template @@ -106,6 +106,43 @@ # environments is to issolate via the top file. #environment: None +##### File Directory Settings ##### +########################################## +# The Salt Minion can redirect all file server operations to a local directory, +# this allows for the same state tree that is on the master to be used if +# coppied completely onto the minion. This is a literal copy of the settings on +# the master but used to reference a local directory on the minion. + +# Set the file client, the client defaults to looking on the master server for +# files, but can be directed to look at the local file directory setting +# defined below by setting it to local. +#file_client: remote + +# The file directory works on environments passed to the minion, each environment +# can have multiple root directories, the subdirectories in the multiple file +# roots cannot match, otherwise the downloaded files will not be able to be +# reliably ensured. A base environment is required to house the top file. +# Example: +# file_roots: +# base: +# - /srv/salt/ +# dev: +# - /srv/salt/dev/services +# - /srv/salt/dev/states +# prod: +# - /srv/salt/prod/services +# - /srv/salt/prod/states +# +# Default: +#file_roots: +# base: +# - /srv/salt + +# The hash_type is the hash to use when discovering the hash of a file in +# the minion directory, the default is md5, but sha1, sha224, sha256, sha384 +# and sha512 are also supported. +#hash_type: md5 + ###### Security settings ##### ########################################### # Enable "open mode", this mode still maintains encryption, but turns off From 221e068057cf3c6e6b58244390e8c2af878cfe12 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 4 Mar 2012 22:01:08 -0700 Subject: [PATCH 222/598] Add the file_client option to the defeault config --- salt/config.py | 1 + salt/fileclient.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/config.py b/salt/config.py index 0a11df3543e9..f73274f6bc58 100644 --- a/salt/config.py +++ b/salt/config.py @@ -127,6 +127,7 @@ def minion_config(path): 'failhard': False, 'autoload_dynamic_modules': True, 'environment': None, + 'file_client': 'remote', 'file_roots': { 'base': ['/srv/salt'], }, diff --git a/salt/fileclient.py b/salt/fileclient.py index 328d9c35129c..a6c96a93801e 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -30,14 +30,14 @@ def get_file_client(opts): ''' - Read in the ``file_server`` option and return the correct type of file + Read in the ``file_client`` option and return the correct type of file server ''' try: return { 'remote': RemoteClient, 'local': LocalClient - }.get(opts['file_server'], 'remote')(opts) + }.get(opts['file_client'], 'remote')(opts) except KeyError: return RemoteClient(opts) From 3e5bc4ff5a31f24588e30fabed98219043348219 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 4 Mar 2012 22:15:06 -0700 Subject: [PATCH 223/598] Add state_top to the minion config --- conf/minion.template | 4 ++++ salt/config.py | 1 + 2 files changed, 5 insertions(+) diff --git a/conf/minion.template b/conf/minion.template index eb2a6b09f73a..e53f597d6fba 100644 --- a/conf/minion.template +++ b/conf/minion.template @@ -105,6 +105,10 @@ # by statically setting it. Remember that the recommended way to manage # environments is to issolate via the top file. #environment: None +# +# If using the local file directory, then the state top file name needs to be +# defined, by default this is top.sls. +#state_top: top.sls ##### File Directory Settings ##### ########################################## diff --git a/salt/config.py b/salt/config.py index f73274f6bc58..1a37b317912d 100644 --- a/salt/config.py +++ b/salt/config.py @@ -127,6 +127,7 @@ def minion_config(path): 'failhard': False, 'autoload_dynamic_modules': True, 'environment': None, + 'state_top': 'top.sls', 'file_client': 'remote', 'file_roots': { 'base': ['/srv/salt'], From 4ee6ac13b4d1aca6e4d424fcaff0a9fa5b48a153 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 4 Mar 2012 22:36:46 -0700 Subject: [PATCH 224/598] Add faulty returns for get_file --- salt/fileclient.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index a6c96a93801e..1f0c482fa683 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -301,8 +301,10 @@ def get_file(self, path, dest='', makedirs=False, env='base'): ''' path = self._check_proto(path) fnd = self._find_file(path, env) + if not fnd['path']: + return '' if not dest: - dest = _cache_loc(path, env) + dest = self._cache_loc(path, env) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: From c752116fd7b965a6043dcd845cd049867d6b0f76 Mon Sep 17 00:00:00 2001 From: blast_hardcheese <blast@hardchee.se> Date: Sun, 4 Mar 2012 18:45:08 -0800 Subject: [PATCH 225/598] Altering render to use _create_loader --- salt/loader.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/salt/loader.py b/salt/loader.py index 399378d0efdd..a4daf3a67952 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -17,7 +17,7 @@ salt_base_path = os.path.dirname(salt.__file__) -def _create_loader(opts, ext_type, tag): +def _create_loader(opts, ext_type, tag, ext_dirs=True, ext_type_dirs=None): '''Creates Loader instance Order of module_dirs: @@ -29,9 +29,11 @@ def _create_loader(opts, ext_type, tag): sys_types = os.path.join(salt_base_path, ext_type) ext_type_types = [] - ext_type_dirs = '%s_dirs' % ext_type - if ext_type_dirs in opts: - ext_type_types.extend(opts[ext_type_dirs]) + if ext_dirs: + if ext_type_dirs == None: + ext_type_dirs = '{0}_dirs'.format(ext_type) + if ext_type_dirs in opts: + ext_type_types.extend(opts[ext_type_dirs]) module_dirs = ext_type_types + [ext_types, sys_types] return Loader(module_dirs, opts, tag) @@ -75,16 +77,7 @@ def render(opts, functions): ''' Returns the render modules ''' - extra_dirs = [ - os.path.join(opts['extension_modules'], - 'renderers') - ] - if 'render_dirs' in opts: - extra_dirs.extend(opts['render_dirs']) - module_dirs = [ - os.path.join(salt_base_path, 'renderers'), - ] + extra_dirs - load = Loader(module_dirs, opts, 'render') + load = _create_loader(opts, 'renderers', 'render', ext_type_dir='render_dirs') pack = {'name': '__salt__', 'value': functions} rend = load.filter_func('render', pack) @@ -101,13 +94,6 @@ def grains(opts): Return the functions for the dynamic grains and the values for the static grains. ''' - extra_dirs = [ - os.path.join(opts['extension_modules'], - 'grains') - ] - module_dirs = [ - os.path.join(salt_base_path, 'grains'), - ] + extra_dirs if not 'grains' in opts: pre_opts = {} salt.config.load_config( @@ -124,7 +110,7 @@ def grains(opts): opts['grains'] = pre_opts['grains'] else: opts['grains'] = {} - load = Loader(module_dirs, opts, 'grain') + load = _create_loader(opts, 'grains', 'grain', ext_dirs=False) grains = load.gen_grains() grains.update(opts['grains']) return grains From 77e813c81e9c472724bba355714f266788f65e1a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 4 Mar 2012 23:01:46 -0700 Subject: [PATCH 226/598] Add grain_pcre to compound matcher, use R@os:Arch.* --- salt/minion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/minion.py b/salt/minion.py index efe46e85421b..70bc4b1ad990 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -667,6 +667,7 @@ def compound_match(self, tgt): log.debug('Compound target received that is not a string') return False ref = {'G': 'grain', + 'R': 'grain_pcre', 'X': 'exsel', 'L': 'list', 'E': 'pcre'} From 98a482e611ee0e28bf4f50b1b6909710aba28983 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Mon, 5 Mar 2012 06:53:58 -0800 Subject: [PATCH 227/598] Fix a typo in the loader refactor pointed out by @archme --- salt/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/loader.py b/salt/loader.py index a4daf3a67952..c40ed1ef8bb9 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -77,7 +77,7 @@ def render(opts, functions): ''' Returns the render modules ''' - load = _create_loader(opts, 'renderers', 'render', ext_type_dir='render_dirs') + load = _create_loader(opts, 'renderers', 'render', ext_type_dirs='render_dirs') pack = {'name': '__salt__', 'value': functions} rend = load.filter_func('render', pack) From 9f97cd1154e87dc30c793b65a762269574d1bce3 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Mon, 5 Mar 2012 08:01:49 -0800 Subject: [PATCH 228/598] Issue #832, check source state files exist before rendering. Bonus removal of call to globals. --- salt/states/file.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index a7c79c8c6948..1511f8f2321a 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -306,6 +306,13 @@ def _py(sfn, name, source, user, group, mode, env, context=None): 'data': trb} +template_registry = { + 'jinja': _jinja, + 'mako': _mako, + 'py': _py, +} + + def symlink(name, target, force=False, makedirs=False): ''' Create a symlink @@ -530,11 +537,15 @@ def managed(name, # then make sure to cpy it down and templatize things. if template and source: sfn = __salt__['cp.cache_file'](source, __env__) - t_key = '_{0}'.format(template) - if t_key in globals(): + if not os.path.exists(sfn): + ret['result'] = False + ret['comment'] = ( + 'File "{sfn}" could not be found').format(sfn=sfn) + if template in template_registry: context_dict = defaults if defaults else {} - if context: context_dict.update(context) - data = globals()[t_key]( + if context: + context_dict.update(context) + data = template_registry[template]( sfn, name, source, From f6ba7d1f851926783259815b53cbc79365e657d7 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 5 Mar 2012 14:08:14 -0700 Subject: [PATCH 229/598] jid == 0 meand the master is not turned on! --- salt/client.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/salt/client.py b/salt/client.py index c3f7397a637b..c1d19d403836 100644 --- a/salt/client.py +++ b/salt/client.py @@ -183,6 +183,9 @@ def cmd( ret, jid=jid, timeout=timeout) + if pup_data['jid'] == 0: + # Failed to connect to the master and send the pub + return {} return self.get_returns(pub_data['jid'], pub_data['minions'], timeout) def cmd_cli( @@ -208,6 +211,9 @@ def cmd_cli( ret, jid=jid, timeout=timeout) + if pup_data['jid'] == 0: + print 'Failed to connect to the Master, is the Salt Master running?' + return {} for fn_ret in self.get_cli_returns(pub_data['jid'], pub_data['minions'], timeout, @@ -240,12 +246,16 @@ def cmd_iter( ret, jid=jid, timeout=timeout) - for fn_ret in self.get_iter_returns(pub_data['jid'], - pub_data['minions'], - timeout): - if not fn_ret: - continue - yield fn_ret + if pup_data['jid'] == 0: + # Failed to connect to the master and send the pub + yield {} + else: + for fn_ret in self.get_iter_returns(pub_data['jid'], + pub_data['minions'], + timeout): + if not fn_ret: + continue + yield fn_ret def cmd_iter_no_block( self, @@ -269,10 +279,14 @@ def cmd_iter_no_block( ret, jid=jid, timeout=timeout) - for fn_ret in self.get_iter_returns(pub_data['jid'], - pub_data['minions'], - timeout): - yield fn_ret + if pup_data['jid'] == 0: + # Failed to connect to the master and send the pub + yield {} + else: + for fn_ret in self.get_iter_returns(pub_data['jid'], + pub_data['minions'], + timeout): + yield fn_ret def cmd_full_return( self, @@ -296,6 +310,9 @@ def cmd_full_return( ret, jid=jid, timeout=timeout) + if pup_data['jid'] == 0: + # Failed to connect to the master and send the pub + return {} return (self.get_returns(pub_data['jid'], pub_data['minions'], timeout)) From e59c9fbd995af4c020a783e4698e9c860184347b Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Mon, 5 Mar 2012 15:21:32 -0800 Subject: [PATCH 230/598] s/pup_data/pub_data/ woof! --- salt/client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/client.py b/salt/client.py index c1d19d403836..cf670cf3d53b 100644 --- a/salt/client.py +++ b/salt/client.py @@ -183,7 +183,7 @@ def cmd( ret, jid=jid, timeout=timeout) - if pup_data['jid'] == 0: + if pub_data['jid'] == 0: # Failed to connect to the master and send the pub return {} return self.get_returns(pub_data['jid'], pub_data['minions'], timeout) @@ -211,7 +211,7 @@ def cmd_cli( ret, jid=jid, timeout=timeout) - if pup_data['jid'] == 0: + if pub_data['jid'] == 0: print 'Failed to connect to the Master, is the Salt Master running?' return {} for fn_ret in self.get_cli_returns(pub_data['jid'], @@ -246,7 +246,7 @@ def cmd_iter( ret, jid=jid, timeout=timeout) - if pup_data['jid'] == 0: + if pub_data['jid'] == 0: # Failed to connect to the master and send the pub yield {} else: @@ -279,7 +279,7 @@ def cmd_iter_no_block( ret, jid=jid, timeout=timeout) - if pup_data['jid'] == 0: + if pub_data['jid'] == 0: # Failed to connect to the master and send the pub yield {} else: @@ -310,7 +310,7 @@ def cmd_full_return( ret, jid=jid, timeout=timeout) - if pup_data['jid'] == 0: + if pub_data['jid'] == 0: # Failed to connect to the master and send the pub return {} return (self.get_returns(pub_data['jid'], From bae51edae43613ea0a00ca61de21dc2e66f9a825 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 5 Mar 2012 16:30:26 -0700 Subject: [PATCH 231/598] change return to yield, more to come --- salt/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/client.py b/salt/client.py index cf670cf3d53b..1a167081e6cd 100644 --- a/salt/client.py +++ b/salt/client.py @@ -213,7 +213,7 @@ def cmd_cli( timeout=timeout) if pub_data['jid'] == 0: print 'Failed to connect to the Master, is the Salt Master running?' - return {} + yield {} for fn_ret in self.get_cli_returns(pub_data['jid'], pub_data['minions'], timeout, From 7b6d666fce88ddd76cfd38f01b2ed53c55f143ce Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 5 Mar 2012 16:35:29 -0700 Subject: [PATCH 232/598] evaluate for the string 0 --- salt/client.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/salt/client.py b/salt/client.py index 1a167081e6cd..78940965fee6 100644 --- a/salt/client.py +++ b/salt/client.py @@ -183,7 +183,7 @@ def cmd( ret, jid=jid, timeout=timeout) - if pub_data['jid'] == 0: + if pub_data['jid'] == '0': # Failed to connect to the master and send the pub return {} return self.get_returns(pub_data['jid'], pub_data['minions'], timeout) @@ -211,18 +211,19 @@ def cmd_cli( ret, jid=jid, timeout=timeout) - if pub_data['jid'] == 0: + if pub_data['jid'] == '0': print 'Failed to connect to the Master, is the Salt Master running?' yield {} - for fn_ret in self.get_cli_returns(pub_data['jid'], - pub_data['minions'], - timeout, - tgt, - expr_form, - verbose): - if not fn_ret: - continue - yield fn_ret + else: + for fn_ret in self.get_cli_returns(pub_data['jid'], + pub_data['minions'], + timeout, + tgt, + expr_form, + verbose): + if not fn_ret: + continue + yield fn_ret def cmd_iter( self, @@ -246,7 +247,7 @@ def cmd_iter( ret, jid=jid, timeout=timeout) - if pub_data['jid'] == 0: + if pub_data['jid'] == '0': # Failed to connect to the master and send the pub yield {} else: @@ -279,7 +280,7 @@ def cmd_iter_no_block( ret, jid=jid, timeout=timeout) - if pub_data['jid'] == 0: + if pub_data['jid'] == '0': # Failed to connect to the master and send the pub yield {} else: @@ -310,7 +311,7 @@ def cmd_full_return( ret, jid=jid, timeout=timeout) - if pub_data['jid'] == 0: + if pub_data['jid'] == '0': # Failed to connect to the master and send the pub return {} return (self.get_returns(pub_data['jid'], From dfab186d9282a011d356aafb597a9390a6e32dc4 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Mon, 5 Mar 2012 15:53:42 -0800 Subject: [PATCH 233/598] Refactor error reporting in states.file --- salt/states/file.py | 257 +++++++++++++++++++------------------------- 1 file changed, 109 insertions(+), 148 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 1511f8f2321a..71e136329b9b 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -159,6 +159,21 @@ def _gen_keep_files(name, require): return list(keep) +def _check_file(ret, name): + ret = True + msg = '' + + if not os.path.isabs(name): + ret = False + msg = ('Specified file {0} is not an absolute' + ' path').format(name) + elif not os.path.exists(name): + ret = False + msg = '{0}: file not found'.format(name) + + return ret, msg + + def _clean_dir(root, keep): ''' Clean out all of the files and directories in a directory (root) while @@ -175,7 +190,7 @@ def _clean_dir(root, keep): real_keep.add(fn_) if fn_ == '/': break - rm_files = [] + for roots, dirs, files in os.walk(root): for name in files: nfn = os.path.join(roots, name) @@ -190,6 +205,12 @@ def _clean_dir(root, keep): return list(removed) +def _error(ret, err_msg): + ret['result'] = False, + ret['comment'] = err_msg + return ret + + def _mako(sfn, name, source, user, group, mode, env, context=None): ''' Render a mako template, returns the location of the rendered file, @@ -338,18 +359,16 @@ def symlink(name, target, force=False, makedirs=False): 'result': True, 'comment': ''} if not os.path.isabs(name): - ret['result'] = False - ret['comment'] = ('Specified file {0} is not an absolute' - ' path').format(name) - return ret + return _error( + ret, 'Specified file {0} is not an absolute path'.format(name)) + if not os.path.isdir(os.path.dirname(name)): if makedirs: _makedirs(name) else: - ret['result'] = False - ret['comment'] = ('Directory {0} for symlink is not present' - .format(os.path.dirname(name))) - return ret + return _error(ret, + ('Directory {0} for symlink is not present' + ) .format(os.path.dirname(name))) if os.path.islink(name): # The link exists, verify that it matches the target if not os.readlink(name) == target: @@ -364,19 +383,15 @@ def symlink(name, target, force=False, makedirs=False): if force: os.remove(name) else: - ret['result'] = False - ret['comment'] = ('File exists where the symlink {0} should be' - .format(name)) - return ret + return _error(ret, ('File exists where the symlink {0} should be' + .format(name))) elif os.path.isdir(name): # It is not a link or a file, it is a dir, error out if force: shutil.rmtree(name) else: - ret['result'] = False - ret['comment'] = ('Directory exists where the symlink {0} ' + return _error(ret, 'Directory exists where the symlink {0} ' 'should be'.format(name)) - return ret if not os.path.exists(name): # The link is not present, make it os.symlink(target, name) @@ -398,10 +413,8 @@ def absent(name): 'result': True, 'comment': ''} if not os.path.isabs(name): - ret['result'] = False - ret['comment'] = ('Specified file {0} is not an absolute' - ' path').format(name) - return ret + return _error(ret, ('Specified file {0} is not an absolute' + ' path').format(name)) if os.path.isfile(name) or os.path.islink(name): try: os.remove(name) @@ -409,9 +422,8 @@ def absent(name): ret['changes']['removed'] = name return ret except: - ret['result'] = False - ret['comment'] = 'Failed to remove file {0}'.format(name) - return ret + return _error(ret, 'Failed to remove file {0}'.format(name)) + elif os.path.isdir(name): try: shutil.rmtree(name) @@ -419,9 +431,8 @@ def absent(name): ret['changes']['removed'] = name return ret except: - ret['result'] = False - ret['comment'] = 'Failed to remove directory {0}'.format(name) - return ret + return _error(ret, 'Failed to remove directory {0}'.format(name)) + ret['comment'] = 'File {0} is not present'.format(name) return ret @@ -495,14 +506,13 @@ def managed(name, 'name': name, 'result': True} if not os.path.isabs(name): - ret['result'] = False - ret['comment'] = ('Specified file {0} is not an absolute' - ' path').format(name) - return ret + return _error( + ret, ('Specified file {0} is not an absolute' + ' path').format(name)) # Gather the source file from the server sfn = '' source_sum = {} - + # If the source is a list then find which file exists if isinstance(source, list): # get the master file list @@ -538,9 +548,8 @@ def managed(name, if template and source: sfn = __salt__['cp.cache_file'](source, __env__) if not os.path.exists(sfn): - ret['result'] = False - ret['comment'] = ( - 'File "{sfn}" could not be found').format(sfn=sfn) + return _error( + ret, ('File "{sfn}" could not be found').format(sfn=sfn)) if template in template_registry: context_dict = defaults if defaults else {} if context: @@ -556,10 +565,10 @@ def managed(name, context_dict ) else: - ret['result'] = False - ret['comment'] = ('Specified template format {0} is not supported' - .format(template)) - return ret + return _error( + ret, ('Specified template format {0} is not supported' + ).format(template)) + if data['result']: sfn = data['data'] hsum = '' @@ -568,63 +577,52 @@ def managed(name, source_sum = {'hash_type': 'md5', 'hsum': hsum} else: - ret['result'] = False - ret['comment'] = data['data'] __clean_tmp(sfn) - return ret + return _error(ret, data['data']) else: # Copy the file down if there is a source if source: if urlparse.urlparse(source).scheme == 'salt': source_sum = __salt__['cp.hash_file'](source, __env__) if not source_sum: - ret['result'] = False - ret['comment'] = 'Source file {0} not found'.format(source) - return ret + return _error( + ret, 'Source file {0} not found'.format(source)) elif source_hash: protos = ['salt', 'http', 'ftp'] if urlparse.urlparse(source_hash).scheme in protos: # The sourc_hash is a file on a server hash_fn = __salt__['cp.cache_file'](source_hash) if not hash_fn: - ret['result'] = False - ret['comment'] = 'Source hash file {0} not found'.format( + return _error( + ret, 'Source hash file {0} not found'.format( source_hash - ) - return ret + )) comps = [] with open(hash_fn, 'r') as hashfile: comps = hashfile.read().split('=') if len(comps) < 2: - ret['result'] = False - ret['comment'] = ('Source hash file {0} contains an ' - ' invalid hash format, it must be in ' - ' the format <hash type>=<hash>').format( - source_hash - ) - return ret + return _error( + ret, ('Source hash file {0} contains an ' + ' invalid hash format, it must be in ' + ' the format <hash type>=<hash>' + ).format(source_hash)) source_sum['hsum'] = comps[1].strip() source_sum['hash_type'] = comps[0].strip() else: # The source_hash is a hash string comps = source_hash.split('=') if len(comps) < 2: - ret['result'] = False - ret['comment'] = ('Source hash file {0} contains an ' - ' invalid hash format, it must be in ' - ' the format <hash type>=<hash>').format( - source_hash - ) - return ret + return _error( + ret, ('Source hash file {0} contains an ' + ' invalid hash format, it must be in ' + ' the format <hash type>=<hash>' + ).format(source_hash)) source_sum['hsum'] = comps[1].strip() source_sum['hash_type'] = comps[0].strip() else: - ret['result'] = False - ret['comment'] = ('Unable to determine upstream hash of' - ' source file {0}').format( - source - ) - return ret + return _error( + ret, ('Unable to determine upstream hash of' + ' source file {0}').format(source)) # Check changes if the target file exists if os.path.isfile(name): @@ -683,9 +681,9 @@ def managed(name, if not sfn: sfn = __salt__['cp.cache_file'](source, __env__) if not sfn: - ret['result'] = False - ret['comment'] = 'Source file {0} not found'.format(source) - return ret + return _error( + ret, 'Source file {0} not found'.format(source)) + # Check to see if the files are bins if _is_bin(sfn) or _is_bin(name): ret['changes']['diff'] = 'Replace binary file' @@ -720,26 +718,22 @@ def managed(name, if not sfn: sfn = __salt__['cp.cache_file'](source, __env__) if not sfn: - ret['result'] = False - ret['comment'] = 'Source file {0} not found'.format(source) - return ret + return ret.error( + ret, 'Source file {0} not found'.format(source)) + if not os.path.isdir(os.path.dirname(name)): if makedirs: _makedirs(name) else: - ret['result'] = False - ret['comment'] = 'Parent directory not present' __clean_tmp(sfn) - return ret + return _error(ret, 'Parent directory not present') else: if not os.path.isdir(os.path.dirname(name)): if makedirs: _makedirs(name) else: - ret['result'] = False - ret['comment'] = 'Parent directory not present' __clean_tmp(sfn) - return ret + return _error(ret, 'Parent directory not present') # Create the file, user-rw-only if mode will be set if mode: cumask = os.umask(384) @@ -850,31 +844,24 @@ def directory(name, 'result': True, 'comment': ''} if not os.path.isabs(name): - ret['result'] = False - ret['comment'] = ('Specified file {0} is not an absolute' - ' path').format(name) - return ret + return _error( + ret, 'Specified file {0} is not an absolute path'.format(name)) if os.path.isfile(name): - ret['result'] = False - ret['comment'] = ('Specified location {0} exists and is a file' - .format(name)) - return ret + return _error( + ret, 'Specified location {0} exists and is a file'.format(name)) if not os.path.isdir(name): # The dir does not exist, make it if not os.path.isdir(os.path.dirname(name)): if makedirs: _makedirs(name) else: - ret['result'] = False - ret['comment'] = 'No directory to create {0} in'.format(name) - return ret + return _error( + ret, 'No directory to create {0} in'.format(name)) if not os.path.isdir(name): _makedirs(name) os.makedirs(name) if not os.path.isdir(name): - ret['result'] = False - ret['comment'] = 'Failed to create directory {0}'.format(name) - return ret + return _error(ret, 'Failed to create directory {0}'.format(name)) # Check permissions perms = {} @@ -1042,19 +1029,16 @@ def recurse(name, 'result': True, 'comment': ''} if not os.path.isabs(name): - ret['result'] = False - ret['comment'] = ('Specified file {0} is not an absolute' - ' path').format(name) - return ret + return _error( + ret, 'Specified file {0} is not an absolute path'.format(name)) + keep = set() # Verify the target directory if not os.path.isdir(name): if os.path.exists(name): # it is not a dir, but it exists - fail out - ret['result'] = False - ret['comment'] = ('The path {0} exists and is not a directory' - .format(name)) - return ret + return _error( + ret, 'The path {0} exists and is not a directory'.format(name)) os.makedirs(name) for fn_ in __salt__['cp.cache_dir'](source, __env__): if not fn_.strip(): @@ -1133,14 +1117,9 @@ def sed(name, before, after, limit='', backup='.bak', options='-r -e', ''' ret = {'name': name, 'changes': {}, 'result': False, 'comment': ''} - if not os.path.isabs(name): - ret['result'] = False - ret['comment'] = ('Specified file {0} is not an absolute' - ' path').format(name) - return ret - if not os.path.exists(name): - ret['comment'] = "File '{0}' not found".format(name) - return ret + check_res, check_msg = _check_file(name) + if not check_res: + return _error(ret, check_msg) # sed returns no output if the edit matches anything or not so we'll have # to look for ourselves @@ -1183,18 +1162,12 @@ def comment(name, regex, char='#', backup='.bak'): ''' ret = {'name': name, 'changes': {}, 'result': False, 'comment': ''} - if not os.path.isabs(name): - ret['result'] = False - ret['comment'] = ('Specified file {0} is not an absolute' - ' path').format(name) - return ret + check_res, check_msg = _check_file(name) + if not check_res: + return _error(ret, check_msg) unanchor_regex = regex.lstrip('^').rstrip('$') - if not os.path.exists(name): - ret['comment'] = "File not found" - return ret - # Make sure the pattern appears in the file before continuing if not __salt__['file.contains'](name, regex): if __salt__['file.contains'](name, unanchor_regex, @@ -1203,8 +1176,7 @@ def comment(name, regex, char='#', backup='.bak'): ret['result'] = True return ret else: - ret['comment'] = "Pattern not found" - return ret + return _error(ret, '{0}: Pattern not found'.format(unanchor_regex)) # Perform the edit __salt__['file.comment'](name, regex, char, backup) @@ -1236,16 +1208,12 @@ def uncomment(name, regex, char='#', backup='.bak'): .. versionadded:: 0.9.5 ''' ret = {'name': name, 'changes': {}, 'result': False, 'comment': ''} - if not os.path.isabs(name): - ret['result'] = False - ret['comment'] = ('Specified file {0} is not an absolute' - ' path').format(name) - return ret - unanchor_regex = regex.lstrip('^') - if not os.path.exists(name): - ret['comment'] = "File not found" - return ret + check_res, check_msg = _check_file(name) + if not check_res: + return _error(ret, check_msg) + + unanchor_regex = regex.lstrip('^') # Make sure the pattern appears in the file if not __salt__['file.contains'](name, unanchor_regex, @@ -1255,8 +1223,7 @@ def uncomment(name, regex, char='#', backup='.bak'): ret['result'] = True return ret else: - ret['comment'] = "Pattern not found" - return ret + return _error(ret, '{0}: Pattern not found'.format(regex)) # Perform the edit __salt__['file.uncomment'](name, regex, char, backup) @@ -1303,11 +1270,9 @@ def append(name, text): ''' ret = {'name': name, 'changes': {}, 'result': False, 'comment': ''} - if not os.path.isabs(name): - ret['result'] = False - ret['comment'] = ('Specified file {0} is not an absolute' - ' path').format(name) - return ret + check_res, check_msg = _check_file(name) + if not check_res: + return _error(ret, check_msg) if isinstance(text, basestring): text = (text,) @@ -1318,8 +1283,7 @@ def append(name, text): except AttributeError: logger.debug("Error appending text to %s; given object is: %s", name, type(chunk)) - ret['comment'] = "Given text is not a string" - return ret + return _error(ret, "Given text is not a string") for line in lines: if __salt__['file.contains'](name, line): @@ -1335,6 +1299,7 @@ def append(name, text): ret['result'] = True return ret + def touch(name, atime=None, mtime=None, makedirs=False): """ Replicate the 'nix "touch" command to create a new empty @@ -1353,18 +1318,14 @@ def touch(name, atime=None, mtime=None, makedirs=False): 'changes': {}, } if not os.path.isabs(name): - ret['result'] = False - ret['comment'] = ('Specified file {0} is not an absolute' - ' path').format(name) - return ret + _error( + ret, 'Specified file {0} is not an absolute path'.format(name)) + if makedirs: _makedirs(name) if not os.path.isdir(os.path.dirname(name)): - ret['result'] = False - ret['comment'] = 'Direcotry not present to touch file {0}'.format( - name - ) - return ret + return _error( + ret, 'Directory not present to touch file {0}'.format(name)) exists = os.path.exists(name) ret['result'] = __salt__['file.touch'](name, atime, mtime) From d96f5107a8331b8e2df4eb0a89615bee69978378 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Mon, 5 Mar 2012 16:17:50 -0800 Subject: [PATCH 234/598] Fix typo --- salt/fileclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index 328d9c35129c..01acd82aca00 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -297,7 +297,7 @@ def _find_file(self, path, env='base'): def get_file(self, path, dest='', makedirs=False, env='base'): ''' - Coppies a file from the local files directory and coppies it into place + Copies a file from the local files directory and coppies it into place ''' path = self._check_proto(path) fnd = self._find_file(path, env) From fe9c9c7637fe82b5b54e0511974749939b22cb27 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Mon, 5 Mar 2012 18:07:58 -0800 Subject: [PATCH 235/598] Check file permissions and user/group ownership after it exists. Also refactor common permission check code --- salt/states/file.py | 179 ++++++++++++++------------------------------ 1 file changed, 55 insertions(+), 124 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 71e136329b9b..94a65a0c1779 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -334,6 +334,55 @@ def _py(sfn, name, source, user, group, mode, env, context=None): } +def _check_perms(name, ret, user, group, mode): + # Check permissions + perms = {} + perms['luser'] = __salt__['file.get_user'](name) + perms['lgroup'] = __salt__['file.get_group'](name) + perms['lmode'] = __salt__['file.get_mode'](name) + + # Mode changes if needed + if mode: + if mode != perms['lmode']: + if not __opts__['test']: + __salt__['file.set_mode'](name, mode) + print __salt__['file.get_mode'](name) + if mode != __salt__['file.get_mode'](name): + ret['result'] = False + ret['comment'] += 'Failed to change mode to {0} '.format(mode) + else: + ret['changes']['mode'] = mode + # user/group changes if needed, then check if it worked + if user: + if user != perms['luser']: + perms['cuser'] = user + if group: + if group != perms['lgroup']: + perms['cgroup'] = group + if 'cuser' in perms or 'cgroup' in perms: + if not __opts__['test']: + __salt__['file.chown']( + name, + user, + group + ) + if user: + if user != __salt__['file.get_user'](name): + ret['result'] = False + ret['comment'] = 'Failed to change user to {0} '.format(user) + elif 'cuser' in perms: + ret['changes']['user'] = user + if group: + if group != __salt__['file.get_group'](name): + ret['result'] = False + ret['comment'] += ('Failed to change group to {0} ' + .format(group)) + elif 'cgroup' in perms: + ret['changes']['group'] = group + + return ret, perms + + def symlink(name, target, force=False, makedirs=False): ''' Create a symlink @@ -626,49 +675,6 @@ def managed(name, # Check changes if the target file exists if os.path.isfile(name): - # Check permissions - perms = {} - perms['luser'] = __salt__['file.get_user'](name) - perms['lgroup'] = __salt__['file.get_group'](name) - perms['lmode'] = __salt__['file.get_mode'](name) - # Mode changes if needed - if mode: - if mode != perms['lmode']: - if not __opts__['test']: - __salt__['file.set_mode'](name, mode) - if mode != __salt__['file.get_mode'](name): - ret['result'] = False - ret['comment'] += 'Mode not changed ' - else: - ret['changes']['mode'] = mode - # user/group changes if needed, then check if it worked - if user: - if user != perms['luser']: - perms['cuser'] = user - if group: - if group != perms['lgroup']: - perms['cgroup'] = group - if 'cuser' in perms or 'cgroup' in perms: - if not __opts__['test']: - __salt__['file.chown']( - name, - user, - group - ) - if user: - if user != __salt__['file.get_user'](name): - ret['result'] = False - ret['comment'] = 'Failed to change user to {0} '.format(user) - elif 'cuser' in perms: - ret['changes']['user'] = user - if group: - if group != __salt__['file.get_group'](name): - ret['result'] = False - ret['comment'] += ('Failed to change group to {0} ' - .format(group)) - elif 'cgroup' in perms: - ret['changes']['group'] = group - # Only test the checksums on files with managed contents if source: name_sum = '' @@ -700,6 +706,8 @@ def managed(name, shutil.copyfile(sfn, name) __clean_tmp(sfn) + ret, perms = _check_perms(name, ret, user, group, mode) + if not ret['comment']: ret['comment'] = 'File {0} updated'.format(name) @@ -741,47 +749,8 @@ def managed(name, os.umask(cumask) ret['changes']['new'] = 'file {0} created'.format(name) ret['comment'] = 'Empty file' - # Check permissions - perms = {} - perms['luser'] = __salt__['file.get_user'](name) - perms['lgroup'] = __salt__['file.get_group'](name) - perms['lmode'] = __salt__['file.get_mode'](name) - # Run through the perms and detect and apply the needed changes to - # permissions - if user: - if user != perms['luser']: - perms['cuser'] = user - if group: - if group != perms['lgroup']: - perms['cgroup'] = group - if 'cuser' in perms or 'cgroup' in perms: - if not __opts__['test']: - __salt__['file.chown']( - name, - user, - group - ) - if mode: - if mode != perms['lmode']: - if not __opts__['test']: - __salt__['file.set_mode'](name, mode) - if mode != __salt__['file.get_mode'](name): - ret['result'] = False - ret['comment'] += 'Mode not changed ' - else: - ret['changes']['mode'] = mode - if user: - if user != __salt__['file.get_user'](name): - ret['result'] = False - ret['comment'] += 'User not changed ' - elif 'cuser' in perms: - ret['changes']['user'] = user - if group: - if group != __salt__['file.get_group'](name): - ret['result'] = False - ret['comment'] += 'Group not changed ' - elif 'cgroup' in perms: - ret['changes']['group'] = group + + ret, perms = _check_perms(name, ret, user, group, mode) # Now copy the file contents if there is a source file if sfn: @@ -864,48 +833,10 @@ def directory(name, return _error(ret, 'Failed to create directory {0}'.format(name)) # Check permissions - perms = {} - perms['luser'] = __salt__['file.get_user'](name) - perms['lgroup'] = __salt__['file.get_group'](name) - perms['lmode'] = __salt__['file.get_mode'](name) - # Run through the perms and detect and apply the needed changes - if user: - if user != perms['luser']: - perms['cuser'] = user - if group: - if group != perms['lgroup']: - perms['cgroup'] = group - if 'cuser' in perms or 'cgroup' in perms: - if not __opts__['test']: - __salt__['file.chown']( - name, - user, - group - ) - if mode: - if mode != perms['lmode']: - if not __opts__['test']: - __salt__['file.set_mode'](name, mode) - if mode != __salt__['file.get_mode'](name): - ret['result'] = False - ret['comment'] += 'Mode not changed ' - else: - ret['changes']['mode'] = mode - if user: - if user != __salt__['file.get_user'](name): - ret['result'] = False - ret['comment'] = 'Failed to change user to {0} '.format(user) - elif 'cuser' in perms: - ret['changes']['user'] = user - if group: - if group != __salt__['file.get_group'](name): - ret['result'] = False - ret['comment'] += 'Failed to change group to {0} '.format(group) - elif 'cgroup' in perms: - ret['changes']['group'] = group + ret, perms = _check_perms(ret, user, group, mode) if recurse: - if not set(['user','group']) >= set(recurse): + if not set(['user', 'group']) >= set(recurse): ret['result'] = False ret['comment'] = 'Types for "recurse" limited to "user" and ' \ '"group"' From 7ad372829082acd9e3b2afde5d26320354a7427f Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Mon, 5 Mar 2012 22:14:22 -0500 Subject: [PATCH 236/598] Only grab group and user out of qemu.conf if they are uncommented. Default to root:root --- salt/modules/virt.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 7bd4812dcebe..1e8f1013308a 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -49,14 +49,20 @@ def _libvirt_creds(): ''' Returns the user and group that the disk images should be owned by ''' - g_cmd = 'grep group /etc/libvirt/qemu.conf' - u_cmd = 'grep user /etc/libvirt/qemu.conf' - group = subprocess.Popen(g_cmd, + g_cmd = 'grep ^group /etc/libvirt/qemu.conf' + u_cmd = 'grep ^user /etc/libvirt/qemu.conf' + try: + group = subprocess.Popen(g_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0].split('"')[1] - user = subprocess.Popen(u_cmd, + except IndexError: + group = "root" + try: + user = subprocess.Popen(u_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0].split('"')[1] + except IndexError: + user = "root" return {'user': user, 'group': group} From 593738863455914ed47ba5918c326501eb50b897 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 5 Mar 2012 21:41:52 -0700 Subject: [PATCH 237/598] Add windows ps grain --- salt/grains/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/grains/core.py b/salt/grains/core.py index 3e6c19ab4f28..94eb06448931 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -257,6 +257,8 @@ def _ps(osdata): bsd_choices = ('FreeBSD', 'NetBSD', 'OpenBSD', 'Darwin') if osdata['os'] in bsd_choices: grains['ps'] = 'ps auxwww' + elif osdata['os'] = 'Windows': + grains['ps'] = 'tasklist.exe' elif osdata.get('virtual', '') == 'openvzhn': grains['ps'] = 'vzps -E 0 -efH|cut -b 6-' else: From 770de8ce85d5ceb09de3e934450840717a408c60 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 5 Mar 2012 21:42:48 -0700 Subject: [PATCH 238/598] fix grains typo --- salt/grains/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 94eb06448931..c0b1af8ecc2a 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -257,7 +257,7 @@ def _ps(osdata): bsd_choices = ('FreeBSD', 'NetBSD', 'OpenBSD', 'Darwin') if osdata['os'] in bsd_choices: grains['ps'] = 'ps auxwww' - elif osdata['os'] = 'Windows': + elif osdata['os'] == 'Windows': grains['ps'] = 'tasklist.exe' elif osdata.get('virtual', '') == 'openvzhn': grains['ps'] = 'vzps -E 0 -efH|cut -b 6-' From 467d54870ee601fc1618e9cc542240705e637999 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 5 Mar 2012 21:51:58 -0700 Subject: [PATCH 239/598] Add _ps function to windows grains --- salt/grains/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/grains/core.py b/salt/grains/core.py index c0b1af8ecc2a..5139415d2f8c 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -341,6 +341,7 @@ def os_data(): grains.update(_memdata(grains)) grains.update(_windows_platform_data(grains)) grains.update(_windows_cpudata()) + grains.update(_ps()) return grains grains.update(_kernel()) From a9a3db6fd00bfab6bc425327bc0e76ec6bbbbd05 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 5 Mar 2012 21:56:41 -0700 Subject: [PATCH 240/598] another fix for windows grains support --- salt/grains/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 5139415d2f8c..18a965bcfd98 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -341,7 +341,7 @@ def os_data(): grains.update(_memdata(grains)) grains.update(_windows_platform_data(grains)) grains.update(_windows_cpudata()) - grains.update(_ps()) + grains.update(_ps(grains)) return grains grains.update(_kernel()) From d0adeb1c92a905b9d4e6de5723474c049392aacd Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Mon, 5 Mar 2012 21:36:05 -0800 Subject: [PATCH 241/598] Fix function signature --- salt/states/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index 94a65a0c1779..1b1a9f236015 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -159,7 +159,7 @@ def _gen_keep_files(name, require): return list(keep) -def _check_file(ret, name): +def _check_file(name): ret = True msg = '' From 4273581253072fd0d2e494f8d8a77896009d5b8b Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Mon, 5 Mar 2012 22:23:54 -0800 Subject: [PATCH 242/598] More use of context managers, check files are being closed when possible --- salt/fileclient.py | 75 +++++++++++++++++++------------------------- salt/modules/file.py | 3 +- salt/state.py | 7 +++-- 3 files changed, 40 insertions(+), 45 deletions(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index 01acd82aca00..395b78bec232 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -28,6 +28,7 @@ log = logging.getLogger(__name__) + def get_file_client(opts): ''' Read in the ``file_server`` option and return the correct type of file @@ -41,6 +42,7 @@ def get_file_client(opts): except KeyError: return RemoteClient(opts) + class Client(object): ''' Base class for Salt file interactions @@ -48,7 +50,7 @@ class Client(object): def __init__(self, opts): self.opts = opts self.serial = salt.payload.Serial(self.opts) - + def _check_proto(self, path): ''' Make sure that this path is intended for the salt master and trim it @@ -75,6 +77,7 @@ def _file_local_list(self, dest): return filelist + @contextlib.contextmanager def _cache_loc(self, path, env='base'): ''' Return the local location to cache the file, cache dirs will be made @@ -86,12 +89,12 @@ def _cache_loc(self, path, env='base'): path ) destdir = os.path.dirname(dest) - cumask = os.umask(191) + cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO) if not os.path.isdir(destdir): os.makedirs(destdir) - os.chmod(dest, 384) + yield dest + os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) os.umask(cumask) - return dest def cache_file(self, path, env='base'): ''' @@ -272,6 +275,7 @@ def get_url(self, url, dest, makedirs=False, env='base'): raise MinionError('Error reading {0}: {1}'.format(url, ex.reason)) return '' + class LocalClient(Client): ''' Use the local_roots option to parse a local file root @@ -302,7 +306,8 @@ def get_file(self, path, dest='', makedirs=False, env='base'): path = self._check_proto(path) fnd = self._find_file(path, env) if not dest: - dest = _cache_loc(path, env) + with self._cache_loc(path, env) as cache_dest: + dest = cache_dest destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: @@ -342,8 +347,8 @@ def file_list_emptydirs(self, env='base'): return ret for path in self.opts['file_roots'][env]: for root, dirs, files in os.walk(path): - if len(dirs)==0 and len(files)==0: - ret.append(os.path.relpath(root,path)) + if len(dirs) == 0 and len(files) == 0: + ret.append(os.path.relpath(root, path)) return ret def hash_file(self, path, env='base'): @@ -362,18 +367,20 @@ def hash_file(self, path, env='base'): log.warning(err) return ret else: - ret['hsum'] = hashlib.md5(open(path, 'rb').read()).hexdigest() + with open(path, 'rb') as f: + ret['hsum'] = hashlib.md5(f.read()).hexdigest() ret['hash_type'] = 'md5' return ret path = self._find_file(path, env)['path'] if not path: return {} ret = {} - ret['hsum'] = getattr(hashlib, self.opts['hash_type'])( - open(path, 'rb').read()).hexdigest() + with open(path, 'rb') as f: + ret['hsum'] = getattr(hashlib, self.opts['hash_type'])( + f.read()).hexdigest() ret['hash_type'] = self.opts['hash_type'] return ret - + def list_env(self, path, env='base'): ''' Return a list of the files in the file server's specified environment @@ -448,6 +455,9 @@ def get_file(self, path, dest='', makedirs=False, env='base'): ''' path = self._check_proto(path) payload = {'enc': 'aes'} + load = {'path': path, + 'env': env, + 'cmd': '_serve_file'} fn_ = None if dest: destdir = os.path.dirname(dest) @@ -457,9 +467,6 @@ def get_file(self, path, dest='', makedirs=False, env='base'): else: return False fn_ = open(dest, 'w+') - load = {'path': path, - 'env': env, - 'cmd': '_serve_file'} while True: if not fn_: load['loc'] = 0 @@ -471,36 +478,19 @@ def get_file(self, path, dest='', makedirs=False, env='base'): if not data['data']: if not fn_ and data['dest']: # This is a 0 byte file on the master - dest = os.path.join( - self.opts['cachedir'], - 'files', - env, - data['dest'] - ) - destdir = os.path.dirname(dest) - cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO) - if not os.path.isdir(destdir): - os.makedirs(destdir) - if not os.path.exists(dest): - open(dest, 'w+').write(data['data']) - os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) - os.umask(cumask) + with self._cache_loc(data['dest'], env) as cache_dest: + dest = cache_dest + if not os.path.exists(cache_dest): + with open(cache_dest, 'w+') as f: + f.write(data['data']) break if not fn_: - dest = os.path.join( - self.opts['cachedir'], - 'files', - env, - data['dest'] - ) - destdir = os.path.dirname(dest) - cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO) - if not os.path.isdir(destdir): - os.makedirs(destdir) - fn_ = open(dest, 'w+') - os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) - os.umask(cumask) + with self._cache_loc(data['dest'], env) as cache_dest: + dest = cache_dest + fn_ = open(dest, 'w+') fn_.write(data['data']) + if fn_: + fn_.close() return dest def file_list(self, env='base'): @@ -541,7 +531,8 @@ def hash_file(self, path, env='base'): return {} else: ret = {} - ret['hsum'] = hashlib.md5(open(path, 'rb').read()).hexdigest() + with open(path, 'rb') as f: + ret['hsum'] = hashlib.md5(f.read()).hexdigest() ret['hash_type'] = 'md5' return ret payload = {'enc': 'aes'} diff --git a/salt/modules/file.py b/salt/modules/file.py index 506d66837fd8..ba16632e986b 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -235,7 +235,8 @@ def get_sum(path, form='md5'): if not os.path.isfile(path): return 'File not found' try: - return getattr(hashlib, form)(open(path, 'rb').read()).hexdigest() + with open(path, 'rb') as f: + return getattr(hashlib, form)(f.read()).hexdigest() except (IOError, OSError), e: return 'File Error: %s' % (str(e)) except AttributeError, e: diff --git a/salt/state.py b/salt/state.py index 24c27eda1556..d196d51bdd75 100644 --- a/salt/state.py +++ b/salt/state.py @@ -532,7 +532,9 @@ def template_shebang(self, template): Check the template shebang line and return the renderer ''' # Open up the first line of the sls template - line = open(template, 'r').readline() + line = '' + with open(template, 'r') as f: + line = f.readline() # Check if it starts with a shebang if line.startswith('#!'): # pull out the shebang data @@ -568,7 +570,8 @@ def compile_template_str(self, template): derived from the template. ''' fn_ = tempfile.mkstemp()[1] - open(fn_, 'w+').write(template) + with open(fn_, 'w+') as f: + f.write(template) high = self.rend[self.template_shebang(fn_)](fn_) os.remove(fn_) return high From 09860b447c50d14a9e9996b734f4de03ad3afdb6 Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Tue, 6 Mar 2012 01:32:33 -0500 Subject: [PATCH 243/598] add XEN support to virt.py --- salt/modules/virt.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 1e8f1013308a..8dfda574a463 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -501,7 +501,7 @@ def virt_type(): def is_kvm_hyper(): ''' - Returns a bool whether or not this node is a hypervisor + Returns a bool whether or not this node is a KVM hypervisor CLI Example:: @@ -512,3 +512,27 @@ def is_kvm_hyper(): if 'kvm_' not in open('/proc/modules').read(): return False return 'libvirtd' in __salt__['cmd.run'](__grains__['ps']) + +def is_xen_hyper(): + ''' + Returns a bool whether or not this node is a XEN hypervisor + + CLI Example:: + + salt '*' virt.is_xen_hyper + ''' + if __grains__['virtual_subtype'] != 'Xen Dom0': + return False + if 'xen_' not in open('/proc/modules').read(): + return False + return 'libvirtd' in __salt__['cmd.run'](__grains__['ps']) + +def is_hyper(): + ''' + Returns a bool whether or not this nos is a hypervisor of any kind + + CLI Example:: + + salt '*' virt.is_hyper + ''' + return is_xen_hyper() or is_kvm_hyper() From 4d26addac01dd693e816f0edbefdd229ee47f5e2 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Mon, 5 Mar 2012 22:43:11 -0800 Subject: [PATCH 244/598] Move other permission check to after file is copied into place --- salt/states/file.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 1b1a9f236015..50a81a5a19eb 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -346,7 +346,6 @@ def _check_perms(name, ret, user, group, mode): if mode != perms['lmode']: if not __opts__['test']: __salt__['file.set_mode'](name, mode) - print __salt__['file.get_mode'](name) if mode != __salt__['file.get_mode'](name): ret['result'] = False ret['comment'] += 'Failed to change mode to {0} '.format(mode) @@ -750,13 +749,13 @@ def managed(name, ret['changes']['new'] = 'file {0} created'.format(name) ret['comment'] = 'Empty file' - ret, perms = _check_perms(name, ret, user, group, mode) - # Now copy the file contents if there is a source file if sfn: shutil.copyfile(sfn, name) __clean_tmp(sfn) + ret, perms = _check_perms(name, ret, user, group, mode) + if not ret['comment']: ret['comment'] = 'File ' + name + ' updated' From f2fbd267e39a6cff3a84e122109d5385a38ad42a Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Tue, 6 Mar 2012 01:43:52 -0500 Subject: [PATCH 245/598] Add error handling for new XEN support --- salt/modules/virt.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 8dfda574a463..a7723cbf92b2 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -509,8 +509,12 @@ def is_kvm_hyper(): ''' if __grains__['virtual'] != 'physical': return False - if 'kvm_' not in open('/proc/modules').read(): - return False + try: + if 'kvm_' not in open('/proc/modules').read(): + return False + except IOError: + # No /proc/modules? Are we on Windows? Or Solaris? + return False return 'libvirtd' in __salt__['cmd.run'](__grains__['ps']) def is_xen_hyper(): @@ -521,10 +525,18 @@ def is_xen_hyper(): salt '*' virt.is_xen_hyper ''' - if __grains__['virtual_subtype'] != 'Xen Dom0': - return False - if 'xen_' not in open('/proc/modules').read(): - return False + try: + if __grains__['virtual_subtype'] != 'Xen Dom0': + return False + except KeyError: + # virtual_subtype isn't set everywhere. + return False + try: + if 'xen_' not in open('/proc/modules').read(): + return False + except IOError: + # No /proc/modules? Are we on Windows? Or Solaris? + return False return 'libvirtd' in __salt__['cmd.run'](__grains__['ps']) def is_hyper(): From 80883d6492f6327394b860385bf45f3f6cb6f8c1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 10:10:27 -0700 Subject: [PATCH 246/598] Fix break in file server A chmod was added before a file was acctually placed --- salt/fileclient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index 9dd5e6d2315c..f44965333e77 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -93,7 +93,6 @@ def _cache_loc(self, path, env='base'): if not os.path.isdir(destdir): os.makedirs(destdir) yield dest - os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) os.umask(cumask) def cache_file(self, path, env='base'): From 883138b8506053d9203a8f9bfc1a035e6f8c4b69 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 10:14:43 -0700 Subject: [PATCH 247/598] Add option to _check_perms --- salt/states/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index 50a81a5a19eb..5c1db18f2ecc 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -832,7 +832,7 @@ def directory(name, return _error(ret, 'Failed to create directory {0}'.format(name)) # Check permissions - ret, perms = _check_perms(ret, user, group, mode) + ret, perms = _check_perms(name, ret, user, group, mode) if recurse: if not set(['user', 'group']) >= set(recurse): From d3a8db8dcb14e778ec34304850da3949b5be6c63 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 11:39:46 -0700 Subject: [PATCH 248/598] Clean the extend data --- salt/state.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/state.py b/salt/state.py index d196d51bdd75..57f9c2d76c60 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1094,6 +1094,18 @@ def render_highstate(self, matches): highstate.update(state) if err: errors += err + # Clean out duplicate extend data + if '__extend__' in highstate: + highext = [] + for ext in highstate['__extend__']: + for key, val in ext.items(): + exists = False + for hext in highext: + if hext == {key: val}: + exists = True + if not exists: + highext.append({key: val}) + highstate['__extend__'] = highext return highstate, errors def call_highstate(self): From 7f7a47b028a5c02996619fefcb93c29206fadf04 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 12:06:40 -0700 Subject: [PATCH 249/598] Add pillar module --- salt/pillar.py | 277 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 salt/pillar.py diff --git a/salt/pillar.py b/salt/pillar.py new file mode 100644 index 000000000000..536ec3177e41 --- /dev/null +++ b/salt/pillar.py @@ -0,0 +1,277 @@ +''' +Render the pillar data +''' + +# Import python libs +import os +import copy +import collections + +# Import Salt libs +import salt.loader +import salt.fileclient +import salt.minion + +class Pillar(object): + ''' + Read over the pillar top files and render the pillar data + ''' + def __init__(self, opts, grains): + # use the local file client + self.opts = copy.deepcopy(opts) + self.opts['file_roots'] = self.opts['pillar_roots'] + self.opts['file_client'] = 'local' + self.opts['grains'] = grains + self.client = salt.fileclient.get_file_client(self.opts) + self.matcher = salt.minion.Matcher(self.opts) + self.rend = salt.loader.render(opts, {}) + + def _get_envs(self): + ''' + Pull the file server environments out of the master options + ''' + envs = set(['base']) + if 'file_roots' in self.opts: + envs.update(self.opts['file_roots'].keys()) + return envs + + def template_shebang(self, template): + ''' + Check the template shebang line and return the renderer + ''' + # Open up the first line of the sls template + line = '' + with open(template, 'r') as f: + line = f.readline() + # Check if it starts with a shebang + if line.startswith('#!'): + # pull out the shebang data + trend = line.strip()[2:] + # If the specified renderer exists, use it, or fallback + if trend in self.rend: + return trend + return self.opts['renderer'] + + def compile_template(self, template, env='', sls=''): + ''' + Take the path to a template and return the high data structure + derived from the template. + ''' + # Template was specified incorrectly + if not isinstance(template, basestring): + return {} + # Template does not exists + if not os.path.isfile(template): + return {} + # Template is an empty file + if salt.utils.is_empty(template): + return {} + # Template is nothing but whitespace + if not open(template).read().strip(): + return {} + + return self.rend[self.template_shebang(template)](template, env, sls) + + def get_tops(self): + ''' + Gather the top files + ''' + tops = collections.defaultdict(list) + include = collections.defaultdict(list) + done = collections.defaultdict(list) + # Gather initial top files + if self.opts['environment']: + tops[self.opts['environment']] = [ + self.compile_template( + self.client.cache_file( + self.opts['state_top'], + self.opts['environment'] + ), + self.opts['environment'] + ) + ] + else: + for env in self._get_envs(): + tops[env].append( + self.compile_template( + self.client.cache_file( + self.opts['state_top'], + env + ), + env + ) + ) + + # Search initial top files for includes + for env, ctops in tops.items(): + for ctop in ctops: + if not 'include' in ctop: + continue + for sls in ctop['include']: + include[env].append(sls) + ctop.pop('include') + # Go through the includes and pull out the extra tops and add them + while include: + pops = [] + for env, states in include.items(): + pops.append(env) + if not states: + continue + for sls in states: + if sls in done[env]: + continue + tops[env].append( + self.compile_template( + self.client.get_state( + sls, + env + ), + env + ) + ) + done[env].append(sls) + for env in pops: + if env in include: + include.pop(env) + + return tops + + def merge_tops(self, tops): + ''' + Cleanly merge the top files + ''' + top = collections.defaultdict(dict) + for sourceenv, ctops in tops.items(): + for ctop in ctops: + for env, targets in ctop.items(): + if env == 'include': + continue + for tgt in targets: + if not tgt in top[env]: + top[env][tgt] = ctop[env][tgt] + continue + matches = [] + states = set() + for comp in ctop[env][tgt]: + if isinstance(comp, dict): + cmatches.append(comp) + if isinstance(comp, basestring): + cstates.add(comp) + for comp in top[env][tgt]: + if isinstance(comp, dict): + matches.append(comp) + if isinstance(comp, basestring): + states.add(comp) + top[env][tgt] = matches + top[env][tgt].extend(list(states)) + return top + + def get_top(self): + ''' + Returns the high data derived from the top file + ''' + tops = self.get_tops() + return self.merge_tops(tops) + + def top_matches(self, top): + ''' + Search through the top high data for matches and return the states + that this minion needs to execute. + + Returns: + {'env': ['state1', 'state2', ...]} + ''' + matches = {} + for env, body in top.items(): + if self.opts['environment']: + if not env == self.opts['environment']: + continue + for match, data in body.items(): + if self.matcher.confirm_top( + match, + data, + self.opts['nodegroups'] + ): + if env not in matches: + matches[env] = [] + for item in data: + if isinstance(item, basestring): + matches[env].append(item) + ext_matches = self.client.ext_nodes() + for env in ext_matches: + if env in matches: + matches[env] = list(set(ext_matches[env]).union(matches[env])) + else: + matches[env] = ext_matches[env] + return matches + + def render_pstate(self, sls, env, mods): + ''' + Collect a single pillar sls file and render it + ''' + err = '' + errors = [] + fn_ = self.client.get_state(sls, env) + if not fn_: + errors.append(('Specified SLS {0} in environment {1} is not' + ' available on the salt master').format(sls, env)) + state = None + try: + state = self.compile_template(fn_, env, sls) + except Exception as exc: + errors.append(('Rendering SLS {0} failed, render error:\n{1}' + .format(sls, exc))) + mods.add(sls) + nstate = None + if state: + if not isinstance(state, dict): + errors.append(('SLS {0} does not render to a dictionary' + .format(sls))) + else: + if 'include' in state: + if not isinstance(state['include'], list): + err = ('Include Declaration in SLS {0} is not formed ' + 'as a list'.format(sls)) + errors.append(err) + else: + for sub_sls in state.pop('include'): + if sub_sls not in mods: + nstate, mods, err = self.render_pstate( + sub_sls, + env, + mods + ) + if nstate: + state.update(nstate) + if err: + errors += err + return state, mods, errors + + def render_pillar(self, matches): + ''' + Extract the sls pillar files from the matches and render them into the + pillar + ''' + pillar = {} + errors = [] + for env, pstates in matches.items(): + mods = set() + for sls in pstates: + pstate, mods, err = self.render_pstate(sls, env, mods) + if pstate: + pillar.update(pstate) + if err: + errors += err + return pillar, errors + + + def compile_pillar(self): + ''' + Render the pillar dta and return + ''' + top = self.get_top() + matches = self.top_matches(top) + pillar, errors = self.render_pillar(matches) + if errors: + return errors + return pillar From 5e4e900d41852d0d6e62d4747b45d6cbd09b6abf Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 12:31:21 -0700 Subject: [PATCH 250/598] Add id to pillar object --- salt/pillar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/pillar.py b/salt/pillar.py index 536ec3177e41..cfec10127a32 100644 --- a/salt/pillar.py +++ b/salt/pillar.py @@ -16,12 +16,13 @@ class Pillar(object): ''' Read over the pillar top files and render the pillar data ''' - def __init__(self, opts, grains): + def __init__(self, opts, grains, id_): # use the local file client self.opts = copy.deepcopy(opts) self.opts['file_roots'] = self.opts['pillar_roots'] self.opts['file_client'] = 'local' self.opts['grains'] = grains + self.opts['id'] = id_ self.client = salt.fileclient.get_file_client(self.opts) self.matcher = salt.minion.Matcher(self.opts) self.rend = salt.loader.render(opts, {}) From cd8dd7bf145c4046f9137cc3c5cdbc3db9740f63 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 12:38:44 -0700 Subject: [PATCH 251/598] Add pillar_roots to the config --- salt/config.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/config.py b/salt/config.py index 1a37b317912d..6b7e36649486 100644 --- a/salt/config.py +++ b/salt/config.py @@ -131,7 +131,10 @@ def minion_config(path): 'file_client': 'remote', 'file_roots': { 'base': ['/srv/salt'], - }, + }, + 'pillar_roots': { + 'base': ['/srv/pillar'], + } 'hash_type': 'md5', 'external_nodes': '', 'disable_modules': [], @@ -198,7 +201,10 @@ def master_config(path): 'cachedir': '/var/cache/salt', 'file_roots': { 'base': ['/srv/salt'], - }, + }, + 'pillar_roots': { + 'base': ['/srv/pillar'], + } 'file_buffer_size': 1048576, 'hash_type': 'md5', 'conf_file': path, From 1c545771cc8c29d0cc01303ab580bbad8bb44d23 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 12:44:59 -0700 Subject: [PATCH 252/598] forgot comma --- salt/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/config.py b/salt/config.py index 6b7e36649486..854a21f88fbf 100644 --- a/salt/config.py +++ b/salt/config.py @@ -134,7 +134,7 @@ def minion_config(path): }, 'pillar_roots': { 'base': ['/srv/pillar'], - } + }, 'hash_type': 'md5', 'external_nodes': '', 'disable_modules': [], @@ -204,7 +204,7 @@ def master_config(path): }, 'pillar_roots': { 'base': ['/srv/pillar'], - } + }, 'file_buffer_size': 1048576, 'hash_type': 'md5', 'conf_file': path, From 62a38a1c90ac1068bc52863ce38799560a9c2c1c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 12:59:42 -0700 Subject: [PATCH 253/598] add state_top reconciliation for pillar --- salt/pillar.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/salt/pillar.py b/salt/pillar.py index cfec10127a32..4e6864d382fd 100644 --- a/salt/pillar.py +++ b/salt/pillar.py @@ -18,15 +18,28 @@ class Pillar(object): ''' def __init__(self, opts, grains, id_): # use the local file client - self.opts = copy.deepcopy(opts) - self.opts['file_roots'] = self.opts['pillar_roots'] - self.opts['file_client'] = 'local' - self.opts['grains'] = grains - self.opts['id'] = id_ + self.opts = self.__gen_opts(opts, grains, id_) self.client = salt.fileclient.get_file_client(self.opts) self.matcher = salt.minion.Matcher(self.opts) self.rend = salt.loader.render(opts, {}) + def __gen_opts(self, opts, grains, id_): + ''' + The options need to be altered to conform to the file client + ''' + opts = copy.deepcopy(opts) + opts['file_roots'] = opts['pillar_roots'] + opts['file_client'] = 'local' + opts['grains'] = grains + opts['id'] = id_ + if opts['state_top'].startswith('salt://'): + opts['state_top'] = opts['state_top'] + elif opts['state_top'].startswith('/'): + opts['state_top'] = os.path.join('salt://', opts['state_top'][1:]) + else: + opts['state_top'] = os.path.join('salt://', opts['state_top']) + return opts + def _get_envs(self): ''' Pull the file server environments out of the master options @@ -191,7 +204,7 @@ def top_matches(self, top): if self.matcher.confirm_top( match, data, - self.opts['nodegroups'] + self.opts.get('nodegroups', {}), ): if env not in matches: matches[env] = [] From 96a1ef866b83f0d571182d1c322bed7571bf1fbc Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 13:21:19 -0700 Subject: [PATCH 254/598] Add pillar module --- salt/modules/pillar.py | 18 ++++++++++++++++++ salt/pillar.py | 13 +++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 salt/modules/pillar.py diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py new file mode 100644 index 000000000000..988595a75157 --- /dev/null +++ b/salt/modules/pillar.py @@ -0,0 +1,18 @@ +''' +Extract the pillar data for this minion +''' + +# Import Salt modules +import salt.pillar + +def data(): + ''' + Returns the pillar derived from the configured pillar source. The pillar + source is derived from the file_client option in the minion config + + CLI Example:: + + salt '*' pillar.data + ''' + pillar = salt.pillar.get_pillar(__opts__, __grains__, __opts__['id']) + return pillar.compile_pillar() diff --git a/salt/pillar.py b/salt/pillar.py index 4e6864d382fd..8addc4ec0cc3 100644 --- a/salt/pillar.py +++ b/salt/pillar.py @@ -12,6 +12,19 @@ import salt.fileclient import salt.minion +def get_pillar(opts, grains, id_): + ''' + Return the correct pillar driver based on the file_client option + ''' + try: + return { + #'remote': RemotePillar, + 'local': Pillar + }.get(opts['file_client'], 'local')(opts, grains, id_) + except KeyError: + return Pillar(opts, grains, id_) + + class Pillar(object): ''' Read over the pillar top files and render the pillar data From effa7c303677e615bf9413db609d490c99a05213 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 14:32:33 -0700 Subject: [PATCH 255/598] Add remote pillar to pillar.py --- salt/pillar.py | 54 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/salt/pillar.py b/salt/pillar.py index 8addc4ec0cc3..f909342d2b33 100644 --- a/salt/pillar.py +++ b/salt/pillar.py @@ -11,32 +11,71 @@ import salt.loader import salt.fileclient import salt.minion +import salt.crypt -def get_pillar(opts, grains, id_): +# Import third party libs +import zmq + +def get_pillar(opts, grains, id_, env=None): ''' Return the correct pillar driver based on the file_client option ''' try: return { - #'remote': RemotePillar, + 'remote': RemotePillar, 'local': Pillar - }.get(opts['file_client'], 'local')(opts, grains, id_) + }.get(opts['file_client'], 'local')(opts, grains, id_, env) except KeyError: - return Pillar(opts, grains, id_) + return Pillar(opts, grains, id_, env) + +class RemotePillar(object): + ''' + Get the pillar from the master + ''' + def __init__(self, opts, grains, id_, env): + self.opts = opts + self.opts['environment'] = env + self.grains = grains + self.id_ = id_ + self.serial = salt.payload.Serial(self.opts) + self.auth = salt.crypt.SAuth(opts) + self.socket = self.__get_socket() + + def __get_socket(self): + ''' + Return the zeromq socket to use + ''' + context = zmq.Context() + socket = context.socket(zmq.REQ) + socket.connect(self.opts['master_uri']) + return socket + + def compile_pillar(self): + ''' + Return the pillar data from the master + ''' + payload = {'enc': 'aes'} + load = {'id': self.id_, + 'grains': self.grains, + 'env': self.opts['environment'], + 'cmd': '_pillar'} + payload['load'] = self.auth.crypticle.dumps(load) + self.socket.send(self.serial.dumps(payload)) + return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) class Pillar(object): ''' Read over the pillar top files and render the pillar data ''' - def __init__(self, opts, grains, id_): + def __init__(self, opts, grains, id_, env): # use the local file client self.opts = self.__gen_opts(opts, grains, id_) self.client = salt.fileclient.get_file_client(self.opts) self.matcher = salt.minion.Matcher(self.opts) self.rend = salt.loader.render(opts, {}) - def __gen_opts(self, opts, grains, id_): + def __gen_opts(self, opts, grains, id_, env=None): ''' The options need to be altered to conform to the file client ''' @@ -45,6 +84,9 @@ def __gen_opts(self, opts, grains, id_): opts['file_client'] = 'local' opts['grains'] = grains opts['id'] = id_ + if 'environment' not in opts: + opts['environment'] = env + opts if opts['state_top'].startswith('salt://'): opts['state_top'] = opts['state_top'] elif opts['state_top'].startswith('/'): From 5f1ed1e9c3ad93377a4f40a41258047a9dd071d7 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 14:33:12 -0700 Subject: [PATCH 256/598] Accept a new pillar command on the master --- salt/master.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/salt/master.py b/salt/master.py index b552a9bf4ada..61128ecfced4 100644 --- a/salt/master.py +++ b/salt/master.py @@ -28,6 +28,7 @@ import salt.utils import salt.client import salt.payload +import salt.pillar log = logging.getLogger(__name__) @@ -536,6 +537,19 @@ def _master_opts(self, load): ''' return self.opts + def _pillar(self, load): + ''' + Return the pillar data for the minion + ''' + if 'id' not in load or 'grains' not in load or 'env' not in load: + return False + pillar = salt.pillar.Pillar( + self.opts, + load['grains'], + load['id'], + load['env']) + return pillar.compile_pillar() + def _return(self, load): ''' Handle the return data sent from the minions From 2c8d19ff9d0ca954a3ee523dfcaa5d89830235c0 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 14:37:12 -0700 Subject: [PATCH 257/598] Add extension_modules to master --- salt/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/config.py b/salt/config.py index 854a21f88fbf..a83cc1360722 100644 --- a/salt/config.py +++ b/salt/config.py @@ -233,9 +233,10 @@ def master_config(path): opts['aes'] = salt.crypt.Crypticle.generate_key_string() + opts['extension_modules'] = os.path.join(opts['cachedir'], 'extmods') # Prepend root_dir to other paths prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', - 'sock_dir', 'key_logfile']) + 'sock_dir', 'key_logfile', 'extension_modules']) # Enabling open mode requires that the value be set to True, and # nothing else! From 97d69856bdeb9d843f531a28cf5cb8bf6466aca6 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 16:07:04 -0700 Subject: [PATCH 258/598] Add pillar data to the minion opts after grains --- salt/config.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/config.py b/salt/config.py index a83cc1360722..51818f14f0ae 100644 --- a/salt/config.py +++ b/salt/config.py @@ -24,6 +24,7 @@ import salt.crypt import salt.loader import salt.utils +import salt.pillar log = logging.getLogger(__name__) @@ -178,6 +179,13 @@ def minion_config(path): opts['grains'] = salt.loader.grains(opts) + opts['pillar'] = salt.pillar.get_pillar( + opts, + opts['grains'], + opts['id'], + opts['environment'], + ).compile_pillar() + # Prepend root_dir to other paths prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', 'key_logfile', 'extension_modules']) From fb25fa04cf553b1084425a1f2af6a9315266ffaf Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 16:32:53 -0700 Subject: [PATCH 259/598] Add pillar data to default renderer --- salt/renderers/yaml_jinja.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/renderers/yaml_jinja.py b/salt/renderers/yaml_jinja.py index 3791013517fd..6d19a16623ef 100644 --- a/salt/renderers/yaml_jinja.py +++ b/salt/renderers/yaml_jinja.py @@ -29,6 +29,7 @@ def render(template_file, env='', sls=''): passthrough = {} passthrough['salt'] = __salt__ passthrough['grains'] = __grains__ + passthrough['pillar'] = __pillar__ passthrough['env'] = env passthrough['sls'] = sls From 8367b242e514382c958cec8fd670399f7e51a6e3 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 16:33:20 -0700 Subject: [PATCH 260/598] Add pillar data to the loader --- salt/loader.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/loader.py b/salt/loader.py index c40ed1ef8bb9..f20c69739419 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -155,6 +155,10 @@ def __init__(self, module_dirs, opts=dict(), tag='module'): self.grains = opts['grains'] else: self.grains = {} + if 'pillar' in opts: + self.pillar = opts['pillar'] + else: + self.pillar = {} self.opts = self.__prep_mod_opts(opts) def __prep_mod_opts(self, opts): @@ -340,6 +344,7 @@ def gen_functions(self, pack=None, virtual_enable=True): mod.__opts__ = self.opts mod.__grains__ = self.grains + mod.__pillar__ = self.pillar if pack: if isinstance(pack, list): From 3e2801d17148a1f6ae62257d1f6b89af65912370 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 16:34:59 -0700 Subject: [PATCH 261/598] Add pillar to json_jinja --- salt/renderers/json_jinja.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/renderers/json_jinja.py b/salt/renderers/json_jinja.py index bfb08809cd18..1c79b27026fa 100644 --- a/salt/renderers/json_jinja.py +++ b/salt/renderers/json_jinja.py @@ -19,6 +19,7 @@ def render(template_file, env='', sls=''): passthrough = {} passthrough['salt'] = __salt__ passthrough['grains'] = __grains__ + passthrough['pillar'] = __pillar__ passthrough['env'] = env passthrough['sls'] = sls From 796f8ab7e051decd7e17fb1d3ed3331842d80d5f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 16:36:04 -0700 Subject: [PATCH 262/598] add pillar to json_mako --- salt/renderers/json_mako.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/renderers/json_mako.py b/salt/renderers/json_mako.py index b5c702a7c867..3a9c698026b4 100644 --- a/salt/renderers/json_mako.py +++ b/salt/renderers/json_mako.py @@ -22,6 +22,7 @@ def render(template): passthrough = {} passthrough['salt'] = __salt__ passthrough['grains'] = __grains__ + passthrough['pillar'] = __pillar__ passthrough['env'] = env passthrough['sls'] = sls From a0991e32546d40d9d074afc5a7888b20d3c50b98 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 16:36:40 -0700 Subject: [PATCH 263/598] Add pillar to yaml_mako --- salt/renderers/yaml_mako.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/renderers/yaml_mako.py b/salt/renderers/yaml_mako.py index a0dda74e6cdb..609532798175 100644 --- a/salt/renderers/yaml_mako.py +++ b/salt/renderers/yaml_mako.py @@ -28,6 +28,7 @@ def render(template, env='', sls=''): passthrough = {} passthrough['salt'] = __salt__ passthrough['grains'] = __grains__ + passthrough['pillar'] = __pillar__ passthrough['env'] = env passthrough['sls'] = sls From 5cab526fde001906a7a08a59d49f738bd45c343b Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 16:37:29 -0700 Subject: [PATCH 264/598] Add pillar to python renderer --- salt/renderers/py.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/renderers/py.py b/salt/renderers/py.py index 1aa581fd543c..4b4a60c30041 100644 --- a/salt/renderers/py.py +++ b/salt/renderers/py.py @@ -22,6 +22,7 @@ def render(template, env='', sls=''): ) mod.salt = __salt__ mod.grains = __grains__ + mod.pillar = __pillar__ mod.env = env mod.sls = sls From 52987c5cf8dcdc8e465dcbc0fd5c1b8f21f613f4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 6 Mar 2012 16:42:18 -0700 Subject: [PATCH 265/598] Add pillar data to templates --- salt/states/file.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/states/file.py b/salt/states/file.py index 5c1db18f2ecc..c679f6a21250 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -230,6 +230,7 @@ def _mako(sfn, name, source, user, group, mode, env, context=None): passthrough = context if context else {} passthrough.update(__salt__) passthrough.update(__grains__) + passthrough.update(__pillar__) with nested(open(sfn, 'r'), open(tgt, 'w+')) as (src, target): template = Template(src.read()) target.write(template.render(**passthrough)) @@ -264,6 +265,7 @@ def _jinja(sfn, name, source, user, group, mode, env, context=None): passthrough = context if context else {} passthrough['salt'] = __salt__ passthrough['grains'] = __grains__ + passthrough['pillar'] = __pillar__ passthrough['name'] = name passthrough['source'] = source passthrough['user'] = user @@ -307,6 +309,7 @@ def _py(sfn, name, source, user, group, mode, env, context=None): ) mod.salt = __salt__ mod.grains = __grains__ + mod.pillar = __pillar__ mod.name = name mod.source = source mod.user = user From f5642c9073895d60a56a2026abc2a07e5c4e7662 Mon Sep 17 00:00:00 2001 From: Grier Johnson <grierj@gmail.com> Date: Tue, 6 Mar 2012 18:26:23 -0800 Subject: [PATCH 266/598] Adding Range Support to Salt Range is a cluster based metadata store. It was used internally at yahoo and recently open source. You can read about range here: https://github.com/grierj/range/wiki/Introduction-to-Range-with-YAML-files Range allows arbitrary grouping of hosts via clusters and the storage of metadata along with them. Additionally there is a powerful DSL that allows for set operations, regexes, and multilayer cluster referencing. The implementation here runs the range query and reduces it to a list style expression format, which means only the salt master needs to know about range or have the range libraries installed. --- conf/master.template | 5 +++++ salt/__init__.py | 2 +- salt/cli/__init__.py | 18 ++++++++++++++++++ salt/cli/cp.py | 2 ++ salt/client.py | 22 ++++++++++++++++++++++ salt/config.py | 1 + 6 files changed, 49 insertions(+), 1 deletion(-) diff --git a/conf/master.template b/conf/master.template index 6105a89a5d86..bec91d681367 100644 --- a/conf/master.template +++ b/conf/master.template @@ -202,3 +202,8 @@ # group1: 'L@foo.domain.com,bar.domain.com,baz.domain.com and bl*.domain.com', # group2: 'G@os:Debian and foo.domain.com', +##### Range Cluster settings ##### +########################################## +# The range server (and optional port) that +# serves your cluster information +#range_server: range:80 diff --git a/salt/__init__.py b/salt/__init__.py index 1e627383b13c..9e23e490f76e 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -171,7 +171,7 @@ def start(self): # Late import so logging works correctly import salt.utils salt.utils.daemonize() - set_pidfile(self.cli['pidfile']) + set_pidfile(self.opts['pidfile']) try: master.start() except salt.master.MasterExit: diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 635c576ed23e..70a3ce712b29 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -112,6 +112,14 @@ def __parse(self): help=('Instead of using shell globs to evaluate the target ' 'use one of the predefined nodegroups to identify a ' 'list of targets.')) + parser.add_option('-R', + '--range', + default=False, + dest='range', + action='store_true', + help=('Instead of using shell globs to evaluate the target ' + 'use a range expressions to identify targets. ' + 'Range expressions look like %cluster')) parser.add_option('-C', '--compound', default=False, @@ -270,6 +278,8 @@ def run(self): args.append('exsel') elif self.opts['nodegroup']: args.append('nodegroup') + elif self.opts['range']: + args.append('range') elif self.opts['compound']: args.append('compound') else: @@ -413,6 +423,14 @@ def __parse(self): help=('Instead of using shell globs to evaluate the target ' 'use one of the predefined nodegroups to identify a ' 'list of targets.')) + parser.add_option('-R', + '--range', + default=False, + dest='range', + action='store_true', + help=('Instead of using shell globs to evaluate the target ' + 'use a range expressions to identify targets. ' + 'Range expressions look like %cluster')) parser.add_option('-c', '--config', default='/etc/salt/master', diff --git a/salt/cli/cp.py b/salt/cli/cp.py index e82f219f42ab..fa2b4d6c9443 100644 --- a/salt/cli/cp.py +++ b/salt/cli/cp.py @@ -78,6 +78,8 @@ def run(self): args.append('grain_pcre') elif self.opts['nodegroup']: args.append('nodegroup') + elif self.opts['range']: + args.append('range') ret = local.cmd(*args) diff --git a/salt/client.py b/salt/client.py index 78940965fee6..583a46e9047e 100644 --- a/salt/client.py +++ b/salt/client.py @@ -44,6 +44,14 @@ import salt.payload from salt.exceptions import SaltClientError, SaltInvocationError +# Try to import range from https://github.com/ytoolshed/range +RANGE = False +try: + import seco.range + RANGE = True +except ImportError: + pass + def prep_jid(cachedir): ''' @@ -150,6 +158,14 @@ def _check_grain_minions(self, expr): ''' return os.listdir(os.path.join(self.opts['pki_dir'], 'minions')) + def _convert_range_to_list(self, tgt): + range = seco.range.Range(self.opts['range_server']) + try: + return range.expand(tgt) + except seco.range.RangeException, e: + print "Range server exception: %s" % e + return [] + def gather_job_info(self, jid, tgt, tgt_type): ''' Return the information about a given job @@ -615,6 +631,12 @@ def pub(self, tgt, fun, arg=(), expr_form='glob', tgt = self.opts['nodegroups'][tgt] expr_form = 'compound' + # Convert a range expression to a list of nodes and change expression + # form to list + if expr_form == 'range' and RANGE: + tgt = self._convert_range_to_list(tgt) + expr_form = 'list' + # Run a check_minions, if no minions match return False # format the payload - make a function that does this in the payload # module diff --git a/salt/config.py b/salt/config.py index 51818f14f0ae..baaeb8e5d688 100644 --- a/salt/config.py +++ b/salt/config.py @@ -229,6 +229,7 @@ def master_config(path): 'pidfile': '/var/run/salt-master.pid', 'cluster_masters': [], 'cluster_mode': 'paranoid', + 'range_server': 'range:80', 'serial': 'msgpack', 'nodegroups': {}, 'key_logfile': '/var/log/salt/key.log', From 75cec7d0dfab7ba78946cd2f8191b4d419ce2440 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Tue, 6 Mar 2012 20:34:02 -0800 Subject: [PATCH 267/598] Use correct var --- salt/utils/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 45c8ab98379e..a01c66601c92 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -142,7 +142,7 @@ def daemonize(): if pid > 0: # exit first parent sys.exit(0) - except OSError as exc: + except OSError as e: msg = 'fork #1 failed: {0} ({1})'.format(e.errno, e.strerror) log.error(msg) sys.exit(1) @@ -157,7 +157,7 @@ def daemonize(): pid = os.fork() if pid > 0: sys.exit(0) - except OSError as exc: + except OSError as e: msg = 'fork #2 failed: {0} ({1})' log.error(msg.format(e.errno, e.strerror)) sys.exit(1) From 3752742fa0632e1c9572a4b1372f56d248f44695 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Tue, 6 Mar 2012 20:42:01 -0800 Subject: [PATCH 268/598] Refactor common template code for pillar and states --- salt/pillar.py | 62 +++++++++++------------------------- salt/state.py | 82 ++++++++++++++---------------------------------- salt/template.py | 57 +++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 103 deletions(-) create mode 100644 salt/template.py diff --git a/salt/pillar.py b/salt/pillar.py index f909342d2b33..edcb30ac9a08 100644 --- a/salt/pillar.py +++ b/salt/pillar.py @@ -13,9 +13,13 @@ import salt.minion import salt.crypt +from salt.template import compile_template + + # Import third party libs import zmq + def get_pillar(opts, grains, id_, env=None): ''' Return the correct pillar driver based on the file_client option @@ -104,43 +108,6 @@ def _get_envs(self): envs.update(self.opts['file_roots'].keys()) return envs - def template_shebang(self, template): - ''' - Check the template shebang line and return the renderer - ''' - # Open up the first line of the sls template - line = '' - with open(template, 'r') as f: - line = f.readline() - # Check if it starts with a shebang - if line.startswith('#!'): - # pull out the shebang data - trend = line.strip()[2:] - # If the specified renderer exists, use it, or fallback - if trend in self.rend: - return trend - return self.opts['renderer'] - - def compile_template(self, template, env='', sls=''): - ''' - Take the path to a template and return the high data structure - derived from the template. - ''' - # Template was specified incorrectly - if not isinstance(template, basestring): - return {} - # Template does not exists - if not os.path.isfile(template): - return {} - # Template is an empty file - if salt.utils.is_empty(template): - return {} - # Template is nothing but whitespace - if not open(template).read().strip(): - return {} - - return self.rend[self.template_shebang(template)](template, env, sls) - def get_tops(self): ''' Gather the top files @@ -151,23 +118,27 @@ def get_tops(self): # Gather initial top files if self.opts['environment']: tops[self.opts['environment']] = [ - self.compile_template( + compile_template( self.client.cache_file( self.opts['state_top'], self.opts['environment'] ), + self.rend, + self.opts['renderer'], self.opts['environment'] ) ] else: for env in self._get_envs(): tops[env].append( - self.compile_template( + compile_template( self.client.cache_file( self.opts['state_top'], env ), - env + self.rend, + self.opts['renderer'], + env=env ) ) @@ -190,12 +161,14 @@ def get_tops(self): if sls in done[env]: continue tops[env].append( - self.compile_template( + compile_template( self.client.get_state( sls, env ), - env + self.rend, + self.opts['renderer'], + env=env ) ) done[env].append(sls) @@ -286,7 +259,8 @@ def render_pstate(self, sls, env, mods): ' available on the salt master').format(sls, env)) state = None try: - state = self.compile_template(fn_, env, sls) + state = compile_template( + fn_, self.rend, self.opts['renderer'], env, sls) except Exception as exc: errors.append(('Rendering SLS {0} failed, render error:\n{1}' .format(sls, exc))) @@ -332,7 +306,7 @@ def render_pillar(self, matches): if err: errors += err return pillar, errors - + def compile_pillar(self): ''' diff --git a/salt/state.py b/salt/state.py index 57f9c2d76c60..760c516a6bfa 100644 --- a/salt/state.py +++ b/salt/state.py @@ -17,7 +17,6 @@ import inspect import fnmatch import logging -import tempfile import collections # Import Salt Libs @@ -26,6 +25,11 @@ import salt.minion import salt.fileclient +from salt.template import ( + compile_template, + compile_template_str, + template_shebang, + ) log = logging.getLogger(__name__) @@ -527,55 +531,6 @@ def reconcile_extend(self, high): high[name][state].append(arg) return high, errors - def template_shebang(self, template): - ''' - Check the template shebang line and return the renderer - ''' - # Open up the first line of the sls template - line = '' - with open(template, 'r') as f: - line = f.readline() - # Check if it starts with a shebang - if line.startswith('#!'): - # pull out the shebang data - trend = line.strip()[2:] - # If the specified renderer exists, use it, or fallback - if trend in self.rend: - return trend - return self.opts['renderer'] - - def compile_template(self, template, env='', sls=''): - ''' - Take the path to a template and return the high data structure - derived from the template. - ''' - # Template was specified incorrectly - if not isinstance(template, basestring): - return {} - # Template does not exists - if not os.path.isfile(template): - return {} - # Template is an empty file - if salt.utils.is_empty(template): - return {} - # Template is nothing but whitespace - if not open(template).read().strip(): - return {} - - return self.rend[self.template_shebang(template)](template, env, sls) - - def compile_template_str(self, template): - ''' - Take the path to a template and return the high data structure - derived from the template. - ''' - fn_ = tempfile.mkstemp()[1] - with open(fn_, 'w+') as f: - f.write(template) - high = self.rend[self.template_shebang(fn_)](fn_) - os.remove(fn_) - return high - def call(self, data): ''' Call a state directly with the low data structure, verify data @@ -776,7 +731,8 @@ def call_template(self, template): ''' Enforce the states in a template ''' - high = self.compile_template(template) + high = compile_template( + template, self.renderers, self.opts['renderer']) if high: return self.call_high(high) return high @@ -785,7 +741,8 @@ def call_template_str(self, template): ''' Enforce the states in a template, pass the template as a string ''' - high = self.compile_template_str(template) + high = compile_template_str( + template, self.renderers, self.opts['renderer']) if high: return self.call_high(high) return high @@ -846,23 +803,27 @@ def get_tops(self): # Gather initial top files if self.opts['environment']: tops[self.opts['environment']] = [ - self.state.compile_template( + compile_template( self.client.cache_file( self.opts['state_top'], self.opts['environment'] ), - self.opts['environment'] + self.state.rend, + self.state.opts['renderer'], + env=self.opts['environment'] ) ] else: for env in self._get_envs(): tops[env].append( - self.state.compile_template( + compile_template( self.client.cache_file( self.opts['state_top'], env ), - env + self.state.rend, + self.state.opts['renderer'], + env=env ) ) @@ -885,12 +846,14 @@ def get_tops(self): if sls in done[env]: continue tops[env].append( - self.state.compile_template( + compile_template( self.client.get_state( sls, env ), - env + self.state.rend, + self.state.opts['renderer'], + env=env ) ) done[env].append(sls) @@ -1006,7 +969,8 @@ def render_state(self, sls, env, mods): ' available on the salt master').format(sls, env)) state = None try: - state = self.state.compile_template(fn_, env, sls) + state = compile_template( + fn_, self.state.rend, self.state.opts['renderer'], env, sls) except Exception as exc: errors.append(('Rendering SLS {0} failed, render error:\n{1}' .format(sls, exc))) diff --git a/salt/template.py b/salt/template.py new file mode 100644 index 000000000000..b803e8f6a498 --- /dev/null +++ b/salt/template.py @@ -0,0 +1,57 @@ +import os.path +import tempfile + +import salt.utils + + +def compile_template(template, renderers, default, env='', sls=''): + ''' + Take the path to a template and return the high data structure + derived from the template. + ''' + # Template was specified incorrectly + if not isinstance(template, basestring): + return {} + # Template does not exists + if not os.path.isfile(template): + return {} + # Template is an empty file + if salt.utils.is_empty(template): + return {} + # Template is nothing but whitespace + with open(template) as f: + if not f.read().strip(): + return {} + return renderers[ + template_shebang(template, renderers, default)](template, env, sls) + + +def compile_template_str(template, renderers, default): + ''' + Take the path to a template and return the high data structure + derived from the template. + ''' + fn_ = tempfile.mkstemp()[1] + with open(fn_, 'w+') as f: + f.write(template) + high = renderers[template_shebang(fn_, renderers, default)](fn_) + os.remove(fn_) + return high + + +def template_shebang(template, renderers, default): + ''' + Check the template shebang line and return the renderer + ''' + # Open up the first line of the sls template + line = '' + with open(template, 'r') as f: + line = f.readline() + # Check if it starts with a shebang + if line.startswith('#!'): + # pull out the shebang data + trend = line.strip()[2:] + # If the specified renderer exists, use it, or fallback + if trend in renderers: + return trend + return default From aafb3813a1135cad7fcf2d6fc24d0fab669ef9ef Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Tue, 6 Mar 2012 20:50:18 -0800 Subject: [PATCH 269/598] Use exc for exception vars to match conventions --- salt/utils/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index a01c66601c92..623e108b4925 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -142,8 +142,8 @@ def daemonize(): if pid > 0: # exit first parent sys.exit(0) - except OSError as e: - msg = 'fork #1 failed: {0} ({1})'.format(e.errno, e.strerror) + except OSError as exc: + msg = 'fork #1 failed: {0} ({1})'.format(exc.errno, exc.strerror) log.error(msg) sys.exit(1) @@ -157,9 +157,9 @@ def daemonize(): pid = os.fork() if pid > 0: sys.exit(0) - except OSError as e: + except OSError as exc: msg = 'fork #2 failed: {0} ({1})' - log.error(msg.format(e.errno, e.strerror)) + log.error(msg.format(exc.errno, exc.strerror)) sys.exit(1) dev_null = open('/dev/null', 'rw') From 1b1c04578e12a6b8955d78d99540a4efd2f8e9a3 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Tue, 6 Mar 2012 21:20:47 -0800 Subject: [PATCH 270/598] Prepend root dirs before loading pillar and grains --- salt/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/config.py b/salt/config.py index 51818f14f0ae..71e40d90d640 100644 --- a/salt/config.py +++ b/salt/config.py @@ -171,12 +171,15 @@ def minion_config(path): # Enabling open mode requires that the value be set to True, and # nothing else! - opts['open_mode'] = opts['open_mode'] is True # set up the extension_modules location from the cachedir opts['extension_modules'] = os.path.join(opts['cachedir'], 'extmods') + # Prepend root_dir to other paths + prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', + 'key_logfile', 'extension_modules']) + opts['grains'] = salt.loader.grains(opts) opts['pillar'] = salt.pillar.get_pillar( @@ -186,9 +189,6 @@ def minion_config(path): opts['environment'], ).compile_pillar() - # Prepend root_dir to other paths - prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', - 'key_logfile', 'extension_modules']) return opts From eeeeab2a04355f6ac744970c5958960d934f5a7f Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Tue, 6 Mar 2012 21:21:12 -0800 Subject: [PATCH 271/598] use calendar module for loading abbreviations from locale --- salt/utils/__init__.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 623e108b4925..f1c644824123 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -2,6 +2,7 @@ Some of the utils used by salt ''' +from calendar import month_abbr as months import os import sys import logging @@ -30,18 +31,6 @@ RED_BOLD = '\033[01;31m' ENDC = '\033[0m' -months = {'01': 'Jan', - '02': 'Feb', - '03': 'Mar', - '04': 'Apr', - '05': 'May', - '06': 'Jun', - '07': 'Jul', - '08': 'Aug', - '09': 'Sep', - '10': 'Oct', - '11': 'Nov', - '12': 'Dec'} def is_empty(filename): ''' @@ -237,7 +226,7 @@ def jid_to_time(jid): ret = '{0}, {1} {2} {3}:{4}:{5}.{6}'.format( year, - months[month], + months[int(month)], day, hour, minute, From 599026db7d2325e56270f2e156a7fadb283569c5 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Tue, 6 Mar 2012 21:37:46 -0800 Subject: [PATCH 272/598] move dns checks into utils --- salt/config.py | 24 +----------------------- salt/minion.py | 23 ++--------------------- salt/utils/__init__.py | 28 +++++++++++++++++++++++++++- 3 files changed, 30 insertions(+), 45 deletions(-) diff --git a/salt/config.py b/salt/config.py index 71e40d90d640..b5a445080ee5 100644 --- a/salt/config.py +++ b/salt/config.py @@ -164,7 +164,7 @@ def minion_config(path): if 'include' in opts: opts = include_config(opts, path) - opts['master_ip'] = dns_check(opts['master']) + opts['master_ip'] = salt.utils.dns_check(opts['master']) opts['master_uri'] = 'tcp://{ip}:{port}'.format(ip=opts['master_ip'], port=opts['master_port']) @@ -253,25 +253,3 @@ def master_config(path): opts['file_roots'] = _validate_file_roots(opts['file_roots']) return opts - -def dns_check(addr): - ''' - Verify that the passed address is valid and return the ipv4 addr if it is - a hostname - ''' - try: - socket.inet_aton(addr) - # is a valid ip addr - except socket.error: - # Not a valid ip addr, check if it is an available hostname - try: - addr = socket.gethostbyname(addr) - except socket.gaierror: - # Woah, this addr is totally bogus, die!!! - err = ('The master address {0} could not be validated, please ' - 'check that the specified master in the minion config ' - 'file is correct\n') - err = err.format(addr) - sys.stderr.write(err) - sys.exit(42) - return addr diff --git a/salt/minion.py b/salt/minion.py index 70bc4b1ad990..4ad74a942b89 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -61,26 +61,6 @@ def get_proc_dir(cachedir): return fn_ -def safe_dns_check(addr): - ''' - Return the ip resolved by dns, but do not exit on failure, only raise an - exception. - ''' - try: - socket.inet_aton(addr) - except socket.error: - # Not a valid ip adder, check DNS - try: - addr = socket.gethostbyname(addr) - except socket.gaierror: - err = ('This master address: {0} was previously resolvable but ' - 'now fails to resolve! The previously resolved ip addr ' - 'will continue to be used').format(addr) - log.error(err) - raise SaltClientError - return addr - - class SMinion(object): ''' Create an object that has loaded all of the minion module functions, @@ -457,7 +437,8 @@ def tune_in(self): if self.opts['dns_check']: try: # Verify that the dns entry has not changed - self.opts['master_ip'] = safe_dns_check(self.opts['master']) + self.opts['master_ip'] = salt.utils.dns_check( + self.opts['master'], safe=True) except SaltClientError: # Failed to update the dns, keep the old addr pass diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index f1c644824123..c14beefad2af 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -3,9 +3,11 @@ ''' from calendar import month_abbr as months +import logging import os +import socket import sys -import logging + log = logging.getLogger(__name__) @@ -249,3 +251,27 @@ def gen_mac(prefix='52:54:'): mac += random.choice(src) + random.choice(src) + ':' return mac[:-1] + +def dns_check(addr, safe=False): + ''' + Return the ip resolved by dns, but do not exit on failure, only raise an + exception. + ''' + try: + socket.inet_aton(addr) + except socket.error: + # Not a valid ip adder, check DNS + try: + addr = socket.gethostbyname(addr) + except socket.gaierror: + err = ('This master address: {0} was previously resolvable but ' + 'now fails to resolve! The previously resolved ip addr ' + 'will continue to be used').format(addr) + if safe: + log.error(err) + raise SaltClientError + else: + err = err.format(addr) + sys.stderr.write(err) + sys.exit(42) + return addr From d3fb1e4461a6994e992a152aadf1e8d26e44a381 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Tue, 6 Mar 2012 21:39:09 -0800 Subject: [PATCH 273/598] remove dead imports --- salt/config.py | 1 - salt/minion.py | 12 ++---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/salt/config.py b/salt/config.py index b5a445080ee5..bd0f41ae0050 100644 --- a/salt/config.py +++ b/salt/config.py @@ -6,7 +6,6 @@ from contextlib import nested import glob import os -import sys import socket import logging import tempfile diff --git a/salt/minion.py b/salt/minion.py index 4ad74a942b89..e75cae5b3391 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -3,24 +3,16 @@ ''' # Import python libs -import BaseHTTPServer -import contextlib + import logging import multiprocessing -import hashlib + import fnmatch import os import re -import shutil -import stat -import string -import socket -import tempfile import threading import time import traceback -import urllib2 -import urlparse # Import zeromq libs import zmq From 768431481b76a9df6f222f1cc7eefb2f6d6fe5cb Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Tue, 6 Mar 2012 21:47:46 -0800 Subject: [PATCH 274/598] Use assigned vars to avoid duplicate lookups --- salt/master.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/master.py b/salt/master.py index 61128ecfced4..089636564136 100644 --- a/salt/master.py +++ b/salt/master.py @@ -329,6 +329,7 @@ def _handle_payload(self, payload): The _handle_payload method is the key method used to figure out what needs to be done with communication to the server ''' + key = load = None try: key = payload['enc'] load = payload['load'] @@ -336,7 +337,7 @@ def _handle_payload(self, payload): return '' return {'aes': self._handle_aes, 'pub': self._handle_pub, - 'clear': self._handle_clear}[payload['enc']](payload['load']) + 'clear': self._handle_clear}[key](load) def _handle_clear(self, load): ''' From e5dcefe82eaae9bbf11311f44ea2a039b3316b0e Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 7 Mar 2012 15:38:43 -0700 Subject: [PATCH 275/598] Add initial pillar docs --- doc/topics/pillar/index.rst | 69 +++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 doc/topics/pillar/index.rst diff --git a/doc/topics/pillar/index.rst b/doc/topics/pillar/index.rst new file mode 100644 index 000000000000..900fe3c4c652 --- /dev/null +++ b/doc/topics/pillar/index.rst @@ -0,0 +1,69 @@ +============== +Pillar of Salt +============== + +Pillar is an interface for Salt designed to offer global values to be +distributed to all minions. Pillar data is managed in a similar way to +the salt state tree. + +Pillar was added to Salt in version 0.9.8 as an experimental add on. + +Declaring the Master Pillar +=========================== + +The Salt Master server maintains a pillar_roots setup that matches the +structure of the file_roots used in the Salt file server. Like the +Salt file server the ``pillar_roots`` option in the master config is based +on environments mapping to directories. The pillar data is then mapped to +minions based on matchers in a top file which is laid out in the same way +as the state top file. + +the configuration for the pillar_roots in the master config is identical in +behavior and function as the file_roots configuration: + +.. code-block:: yaml + + pillar_roots: + base: + - /srv/pillar + +This example configuration declares that the base environment will be located +in the /srv/pillar directory. The top file used matches the name of the top file +used for states, and has the same structure: + +.. code-block:: yaml + + base: + '*': + - packages + +This sim[ple pillar top file declares that information for all minions can be +found in the packages sls file: + +.. code-block:: yaml + + {% if grains['os'] == 'RedHat' %} + apache: httpd + git: git + {% endif %} + {% elif grains['os'] == 'Debian' %} + apache: apache2 + git: git-core + {% endif %} + +Now this data can be used from within modules, renderers, state sls files and +more via the shared pillar dict: + +.. code-block:: yaml + + apache: + pkg: + - installed + - name: {{ pillar['apache'] }} + +.. code-block:: yaml + + git: + pkg: + - installed + - name: {{ pillar['git'] }} From 1adce25093f444b358f951403c5781ba9e59057f Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Wed, 7 Mar 2012 18:06:46 -0800 Subject: [PATCH 276/598] Delay pillar initialization and compilation until minion is authenticated --- salt/config.py | 7 ------- salt/minion.py | 6 ++++++ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/salt/config.py b/salt/config.py index bd0f41ae0050..cce4056cfdda 100644 --- a/salt/config.py +++ b/salt/config.py @@ -181,13 +181,6 @@ def minion_config(path): opts['grains'] = salt.loader.grains(opts) - opts['pillar'] = salt.pillar.get_pillar( - opts, - opts['grains'], - opts['id'], - opts['environment'], - ).compile_pillar() - return opts diff --git a/salt/minion.py b/salt/minion.py index e75cae5b3391..7741b51fbd43 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -96,6 +96,12 @@ def __init__(self, opts): else: log.warn('Starting the Salt Minion') self.authenticate() + opts['pillar'] = salt.pillar.get_pillar( + opts, + opts['grains'], + opts['id'], + opts['environment'], + ).compile_pillar() def __prep_mod_opts(self): ''' From 0ea6595e129872a08409c1c7b0c7cf6ef42522c3 Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Wed, 7 Mar 2012 22:43:30 -0700 Subject: [PATCH 277/598] Add Windows support with win_network.py --- salt/modules/win_network.py | 114 +++++++++++++++--------------------- 1 file changed, 46 insertions(+), 68 deletions(-) diff --git a/salt/modules/win_network.py b/salt/modules/win_network.py index d70fc532801f..ccfc2edccc52 100644 --- a/salt/modules/win_network.py +++ b/salt/modules/win_network.py @@ -22,7 +22,6 @@ def __virtual__(): return False - def _sanitize_host(host): ''' Sanitize host string. @@ -92,6 +91,8 @@ def traceroute(host): continue comps = line.split() complength = len(comps) + # This method still needs to better catch rows of other lengths + # For example if some of the ms returns are '*' if complength == 9: result = { 'count': comps[0], @@ -139,11 +140,9 @@ def nslookup(host): if ":" in line: comps = line.split(":") ret.append({comps[0].strip(): comps[1].strip()}) - return ret - def dig(host): ''' Performs a DNS lookup with dig @@ -203,68 +202,44 @@ def interfaces(): salt '*' network.interfaces ''' ret = {} - - out = __salt__['cmd.run']('ip addr show') - groups = re.compile('\r?\n\d').split(out) - - for group in groups: - iface = None - data = {} - for line in group.split('\n'): - if not ' ' in line: - continue - m = re.match('^\d*:\s+(\w+)(?:@)?(\w+)?:\s+<(.+)>', line) - if m: - iface,parent,attrs = m.groups() - if 'UP' in attrs.split(','): - data['up'] = True - if parent: - data['parent'] = parent + ifaces = [] + lines = __salt__['cmd.run']('ipconfig /all').split('\r\n') + configstart = 0 + configname = '' + for line in lines: + if configstart == 3: + ifaces.append(config) + configstart = 0 + continue + if not line: + configstart = configstart + 1 + continue + if line.startswith(' '): + comps = line.split(':', 1) + config[configname][comps[0].rstrip(' .').strip()] = comps[1].strip() + continue + if configstart == 1: + configname = line.strip(' :') + config = {configname: {}} + configstart = configstart + 1 + continue + for iface in ifaces: + for key, val in iface.iteritems(): + item = {} + itemdict = {'Physical Address': 'hwaddr', + 'IPv4 Address': 'ipaddr', + 'Link-local IPv6 Address': 'ipaddr6', + 'Subnet Mask': 'netmask', + } + item['broadcast'] = None + for k, v in itemdict.iteritems(): + if k in val: + item[v] = val[k].rstrip('(Preferred)') + if 'IPv4 Address' in val: + item['up'] = True else: - cols = line.split() - if len(cols) >= 2: - type,value = tuple(cols[0:2]) - if type in ('inet', 'inet6'): - def parse_network(): - """ - Return a tuple of ip, netmask, broadcast - based on the current set of cols - """ - brd = None - ip,cidr = tuple(value.split('/')) - if type == 'inet': - mask = _cidr_to_ipv4_netmask(int(cidr)) - if 'brd' in cols: - brd = cols[cols.index('brd')+1] - elif type == 'inet6': - mask = cidr - return (ip, mask, brd) - - if 'secondary' not in cols: - ipaddr, netmask, broadcast = parse_network() - if type == 'inet': - data['ipaddr'] = ipaddr - data['netmask'] = netmask - data['broadcast'] = broadcast - elif type == 'inet6': - data['ipaddr6'] = ipaddr - data['netmask6'] = netmask - else: - if 'secondary' not in data: - data['secondary'] = [] - ip, mask, brd = parse_network() - data['secondary'].append({ - 'type': type, - 'ipaddr': ip, - 'netmask': mask, - 'broadcast': brd - }) - del ip, mask, brd - elif type.startswith('link'): - data['hwaddr'] = value - if iface: - ret[iface] = data - del iface, data + item['up'] = False + ret[key] = item return ret @@ -274,7 +249,7 @@ def up(interface): CLI Example:: - salt '*' network.up eth0 + salt '*' network.up 'Wireless LAN adapter Wireless Network Connection' ''' data = interfaces().get(interface) if data: @@ -282,13 +257,14 @@ def up(interface): else: return None + def ipaddr(interface): ''' Returns the IP address for a given interface CLI Example:: - salt '*' network.ipaddr eth0 + salt '*' network.ipaddr 'Wireless LAN adapter Wireless Network Connection' ''' data = interfaces().get(interface) if data: @@ -296,13 +272,14 @@ def ipaddr(interface): else: return None + def netmask(interface): ''' Returns the netmask for a given interface CLI Example:: - salt '*' network.netmask eth0 + salt '*' network.netmask 'Wireless LAN adapter Wireless Network Connection' ''' data = interfaces().get(interface) if data: @@ -310,13 +287,14 @@ def netmask(interface): else: return None + def hwaddr(interface): ''' Returns the hwaddr for a given interface CLI Example:: - salt '*' network.hwaddr eth0 + salt '*' network.hwaddr 'Wireless LAN adapter Wireless Network Connection' ''' data = interfaces().get(interface) if data: From 372fd13ebeee1feca50c67b6924d9f57a413ff19 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 8 Mar 2012 00:16:01 -0700 Subject: [PATCH 278/598] Add file_roots overlay docs --- doc/ref/file_server/file_roots.rst | 36 +++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/doc/ref/file_server/file_roots.rst b/doc/ref/file_server/file_roots.rst index 0f71c50b133c..724ef79a8925 100644 --- a/doc/ref/file_server/file_roots.rst +++ b/doc/ref/file_server/file_roots.rst @@ -3,14 +3,14 @@ File Server Configuration ========================= The Salt file server is a high performance file server written in ZeroMQ. It -manages large files quickly and with little overhead, and has been optomized -to handle small files in an extreamly efficient manner. +manages large files quickly and with little overhead, and has been optimized +to handle small files in an extremely efficient manner. The Salt file server is an environment aware file server, this means that files can be allocated within many root directories and accessed by specifying both the file path and the environment to search. The -individual environments can also be spanned across mutiple directory roots -to crate overlays and to allow for files to be orginaized in many flexible +individual environments can also be spanned across multiple directory roots +to crate overlays and to allow for files to be organized in many flexible ways. Environments @@ -20,7 +20,33 @@ The Salt file server defaults to the mandatory ``base`` environment. This environment MUST be defined and is used to download files when no environment is specified. -Environments allow for files and sls data to be logically seperated, but +Environments allow for files and sls data to be logically separated, but environments are not isolated from each other. This allows for logical isolation of environments by the engineer using Salt, but also allows for information to be used in multiple environments for maximum flexibility. + + +Directory Overlay +================= + +The environment setting is a list of directories to publish files from. These +directories are searched in order to find the specified file and the first file +found is returned. + +This means that directory data is prioritized based on the order in which they +are listed. In the case of this ``file_roots`` configuration: + +.. code-block:: yaml + + file_roots: + base: + - /srv/salt/base + - /srv/salt/failover + +If a file uri os salt://httpd/httpd.conf will first search for the file at +/srv/salt/base/httpd/httpd.conf, if the file is found there it will be +returned, if the file is not found there, then +/srv/salt/failover/httpd/httpd.conf is searched for the file. + +This allows for directories to be overlaid and prioritized based on the order +they are defined in the configuration. From 93f20c12d71f26a56f34ac19959cf645576bad6f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 8 Mar 2012 12:45:47 -0700 Subject: [PATCH 279/598] Catch non-existant functions on the master --- salt/master.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/master.py b/salt/master.py index 45b12266b008..84a1aa838b83 100644 --- a/salt/master.py +++ b/salt/master.py @@ -729,7 +729,12 @@ def run_func(self, func, load): if func.startswith('__'): return self.crypticle.dumps({}) # Run the func - ret = getattr(self, func)(load) + try: + ret = getattr(self, func)(load) + except AttributeError as exc: + log.error(('Recived function {0} which in unavailable on the ' + 'master, returning False').format(exc)) + return self.crypticle.dumps(False) # Don't encrypt the return value for the _return func # (we don't care about the return value, so why encrypt it?) if func == '_return': From 4a3024f09ff0434fd2d99226bd4d4585ea009313 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 8 Mar 2012 12:06:15 -0800 Subject: [PATCH 280/598] Fix a small log message tyop in the previous commit. --- salt/master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/master.py b/salt/master.py index 84a1aa838b83..f987ade8c07f 100644 --- a/salt/master.py +++ b/salt/master.py @@ -732,7 +732,7 @@ def run_func(self, func, load): try: ret = getattr(self, func)(load) except AttributeError as exc: - log.error(('Recived function {0} which in unavailable on the ' + log.error(('Received function {0} which in unavailable on the ' 'master, returning False').format(exc)) return self.crypticle.dumps(False) # Don't encrypt the return value for the _return func From 06c1d1d00086b882277fa94a36d92c2ba179953c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 8 Mar 2012 13:12:53 -0700 Subject: [PATCH 281/598] Add local file server docs --- doc/ref/file_server/file_roots.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/ref/file_server/file_roots.rst b/doc/ref/file_server/file_roots.rst index 724ef79a8925..96edaa6b64ef 100644 --- a/doc/ref/file_server/file_roots.rst +++ b/doc/ref/file_server/file_roots.rst @@ -50,3 +50,13 @@ returned, if the file is not found there, then This allows for directories to be overlaid and prioritized based on the order they are defined in the configuration. + +Local File Server +================= + +The file server can be rerouted to run from the minion. This is primarily to +enable running salt states without a salt master. To use the local file server +interface, copy the file server data to the minion and set the file_roots +option on the minion to point to the directories copied from the master. +Once the minion ``file_roots`` option has been set, change the ``file_client`` +option to local to make sure that the local file server interface is used. From 738a74f6ee398ce434e51175471a23efc8500ff2 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 8 Mar 2012 15:16:19 -0700 Subject: [PATCH 282/598] iAdd pillar_roots to the minion config --- conf/minion.template | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conf/minion.template b/conf/minion.template index e53f597d6fba..7fac17c528f8 100644 --- a/conf/minion.template +++ b/conf/minion.template @@ -147,6 +147,13 @@ # and sha512 are also supported. #hash_type: md5 +# The Salt pillar is searched for locally if file_client is set to local. If +# this is the case, and pillar data is defined, then the pillar_roots need to +# also be configured on the minion: +#pillar_roots: +# base: +# - /srv/pillar + ###### Security settings ##### ########################################### # Enable "open mode", this mode still maintains encryption, but turns off From c09d8547b481eaef75fd60e65f653937588113cc Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 8 Mar 2012 15:26:44 -0700 Subject: [PATCH 283/598] Add pillar_roots to the master config --- conf/master.template | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/conf/master.template b/conf/master.template index bec91d681367..dd101ae4dcfd 100644 --- a/conf/master.template +++ b/conf/master.template @@ -113,6 +113,19 @@ # The buffer size in the file server can be adjusted here: #file_buffer_size: 1048576 +# Pillar Configurations: +# The Salt Pillar, is a system that allows for the building of global data +# that is refined based on minion. Basically, the pillar creates data that +# can be generated to be specific based on the grains of the minion. Pillar +# is laid out in the same fashion as the file server, with environments, a top +# file and sls files. The difference is that the data does not need to be +# in the highstate format, and is generally just key/value pairs. +# +#pillar_roots: +# base: +# - /srv/pillar +# + ##### Syndic settings ##### ########################################## # The Salt syndic is used to pass commands through a master from a higher @@ -127,7 +140,7 @@ #order_masters: False # # If this master will be running a salt syndic daemon, syndic_master tells -# this master where to recieve commands from. +# this master where to receive commands from. #syndic_master: masterofmaster ##### Peer Publish settings ##### From 1250341fe5b051ba8b023c672b747e7271df0c0f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 8 Mar 2012 16:05:34 -0700 Subject: [PATCH 284/598] Add directory check to file state --- salt/states/file.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/states/file.py b/salt/states/file.py index c679f6a21250..89295aa94c36 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -564,6 +564,11 @@ def managed(name, sfn = '' source_sum = {} + if os.path.isdir(name): + ret['comment'] = 'Specified target {0} is a directory'.format(name) + ret['result'] = False + return ret + # If the source is a list then find which file exists if isinstance(source, list): # get the master file list From da9d235fa89e5c6b40252210fe747824a4d61f06 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 8 Mar 2012 16:34:57 -0700 Subject: [PATCH 285/598] Fix issues where sysctl only supported one entry --- salt/modules/linux_sysctl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/modules/linux_sysctl.py b/salt/modules/linux_sysctl.py index 7b6c0e515815..7056fce6f3df 100644 --- a/salt/modules/linux_sysctl.py +++ b/salt/modules/linux_sysctl.py @@ -112,6 +112,8 @@ def persist(name, value, config='/etc/sysctl.conf'): nlines.append('{0} = {1}\n'.format(name, value)) edited = True continue + else: + nlines.append(line) if not edited: nlines.append('{0} = {1}\n'.format(name, value)) open(config, 'w+').writelines(nlines) From 94b1b140525635e95d39212ca099ee9f2706460e Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Thu, 8 Mar 2012 15:52:22 -0800 Subject: [PATCH 286/598] Issue #751, match append text without using regex. By escaping regex patterns in text we can strictly match text in append. --- salt/modules/file.py | 21 +++++++++++++-------- salt/states/file.py | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index ba16632e986b..a852a6ddfbe7 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -351,14 +351,19 @@ def find(path, *opts): ret.sort() return ret -def _sed_esc(s): +def _sed_esc(s, escape_all=False): ''' Escape single quotes and forward slashes ''' - return '{0}'.format(s).replace("'", "'\"'\"'").replace("/", "\/") + special_chars = "^.[$()|*+?{" + s = s.replace("'", "'\"'\"'").replace("/", "\/") + if escape_all: + for ch in special_chars: + s = s.replace(ch, "\\" + ch) + return s def sed(path, before, after, limit='', backup='.bak', options='-r -e', - flags='g'): + flags='g', escape_all=False): ''' Make a simple edit to a file @@ -394,9 +399,9 @@ def sed(path, before, after, limit='', backup='.bak', options='-r -e', .. versionadded:: 0.9.5 ''' # Largely inspired by Fabric's contrib.files.sed() - - before = _sed_esc(before) - after = _sed_esc(after) + # XXX:dc: Do we really want to always force escaping? + before = _sed_esc(before, escape_all) + after = _sed_esc(after, escape_all) cmd = r"sed {backup}{options} '{limit}s/{before}/{after}/{flags}' {path}".format( backup = '-i{0} '.format(backup) if backup else '', @@ -484,7 +489,7 @@ def comment(path, regex, char='#', backup='.bak'): after=r'{0}\1'.format(char), backup=backup) -def contains(path, text, limit=''): +def contains(path, text, limit='', escape=False): ''' Return True if the file at ``path`` contains ``text`` @@ -500,7 +505,7 @@ def contains(path, text, limit=''): return False result = __salt__['file.sed'](path, text, '&', limit=limit, backup='', - options='-n -r -e', flags='gp') + options='-n -r -e', flags='gp', escape_all=escape) return bool(result) diff --git a/salt/states/file.py b/salt/states/file.py index c679f6a21250..6ca06475ca67 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -1219,7 +1219,7 @@ def append(name, text): return _error(ret, "Given text is not a string") for line in lines: - if __salt__['file.contains'](name, line): + if __salt__['file.contains'](name, line, escape=True): continue else: __salt__['file.append'](name, line) From 84aac553322943db7e93ef9651ccfe076bde6635 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 8 Mar 2012 17:20:53 -0800 Subject: [PATCH 287/598] Disable the ps salt module on python 2.6 There are too many issues with it. Fixes #712 --- salt/modules/ps.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/salt/modules/ps.py b/salt/modules/ps.py index bfe979ab0859..5b88c081e78b 100644 --- a/salt/modules/ps.py +++ b/salt/modules/ps.py @@ -4,8 +4,23 @@ ''' import time -import psutil - +try: + import psutil + has_psutil = True +except ImportError: + has_psutil = False + +def __virtual__(): + if not has_psutil: + return False + + # The python 2.6 version of psutil lacks several functions + # used in this salt module so instead of spaghetti string + # code to try to bring sanity to everything, disable it. + if sys.version_info[0] == 2 and sys.version_info[1] < 7: + return False + + return "ps" def top(num_processes=5, interval=3): ''' From ed891146374d64c6eb0886cf6bc72d285bedd692 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 8 Mar 2012 17:24:33 -0800 Subject: [PATCH 288/598] Sorry for breaking the internets --- salt/modules/ps.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/modules/ps.py b/salt/modules/ps.py index 5b88c081e78b..11afcfae0ca9 100644 --- a/salt/modules/ps.py +++ b/salt/modules/ps.py @@ -3,7 +3,9 @@ See http://code.google.com/p/psutil. ''' +import sys import time + try: import psutil has_psutil = True From 238437e6e8195023774d6b69133b537d65b087cd Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Thu, 8 Mar 2012 22:12:25 -0500 Subject: [PATCH 289/598] fixup butterkvm.py to properly find group and user in qemu.conf, set to root if not defined --- salt/modules/butterkvm.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/salt/modules/butterkvm.py b/salt/modules/butterkvm.py index c07bfbf09d68..7d5bea3f572e 100644 --- a/salt/modules/butterkvm.py +++ b/salt/modules/butterkvm.py @@ -103,14 +103,20 @@ def libvirt_creds(): salt '*' butterkvm.libvirt_creds ''' - g_cmd = 'grep group /etc/libvirt/qemu.conf' - u_cmd = 'grep user /etc/libvirt/qemu.conf' - group = subprocess.Popen(g_cmd, + g_cmd = 'grep ^\s*group /etc/libvirt/qemu.conf' + u_cmd = 'grep ^\s*user /etc/libvirt/qemu.conf' + try: + group = subprocess.Popen(g_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0].split('"')[1] - user = subprocess.Popen(u_cmd, + except IndexError: + group = "root" + try: + user = subprocess.Popen(u_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0].split('"')[1] + except IndexError: + user = "root" return {'user': user, 'group': group} From 90fac59ae55072d8b1ba9cc85d44d751fb319ee5 Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Thu, 8 Mar 2012 22:13:02 -0500 Subject: [PATCH 290/598] fixup virt.py to support spaces in front of user and group definitions in qemu.conf --- salt/modules/virt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index a7723cbf92b2..940ee6b0e103 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -49,8 +49,8 @@ def _libvirt_creds(): ''' Returns the user and group that the disk images should be owned by ''' - g_cmd = 'grep ^group /etc/libvirt/qemu.conf' - u_cmd = 'grep ^user /etc/libvirt/qemu.conf' + g_cmd = 'grep ^\s*group /etc/libvirt/qemu.conf' + u_cmd = 'grep ^\s*user /etc/libvirt/qemu.conf' try: group = subprocess.Popen(g_cmd, shell=True, From f4af57831454a8619092a2b10483663b00d2e324 Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Thu, 8 Mar 2012 22:22:11 -0500 Subject: [PATCH 291/598] fixup comments in virt.py --- salt/modules/virt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 940ee6b0e103..42b6420fefd7 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -30,7 +30,7 @@ def __get_conn(): Detects what type of dom this node is and attempts to connect to the correct hypervisor via libvirt. ''' - # This only supports kvm right now, it needs to be expanded to support + # This has only been tested on kvm and xen, it needs to be expanded to support # all vm layers supported by libvirt return libvirt.open("qemu:///system") From 3a5bb0407d5649ace8b5b7af65d02a61ecb01a8c Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Thu, 8 Mar 2012 21:05:52 -0700 Subject: [PATCH 292/598] Various Doc fixes. Mainly small typo fixes and clarifying some language --- doc/ref/configuration/master.rst | 31 ++++++++++++++--------------- doc/ref/states/index.rst | 17 ++++++++-------- doc/topics/installation/index.rst | 2 +- doc/topics/roadmap/index.rst | 4 ++-- doc/topics/tutorials/states_pt1.rst | 4 ++-- doc/topics/tutorials/states_pt3.rst | 14 ++++++------- 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 00abe137d40c..122a348106d5 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -91,7 +91,7 @@ execution returns and command executions. Default: :file:`/` -The system root direcotry to oporate from, change this to make Salt run from +The system root direcotry to operate from, change this to make Salt run from an alternative root .. code-block:: yaml @@ -158,9 +158,9 @@ Open mode is a dangerous security feature. One problem encountered with pki authentication systems is that keys can become "mixed up" and authentication begins to fail. Open mode turns off authentication and tells the master to accept all authentication. This will clean up the pki keys received from the -minions. Open mode should not be turned on for general use, open mode should +minions. Open mode should not be turned on for general use. Open mode should only be used for a short period of time to clean up pki keys. To turn on open -mode the value passed must be ``True``. +mode set this value to ``True``. .. code-block:: yaml @@ -173,7 +173,7 @@ mode the value passed must be ``True``. Default: ``False`` -Enable auto_accept, this setting will automatically accept all incoming +Enable auto_accept. This setting will automatically accept all incoming public keys from the minions .. code-block:: yaml @@ -252,10 +252,10 @@ Default: ``base: [/srv/salt]`` Salt runs a lightweight file server written in zeromq to deliver files to minions. This file server is built into the master daemon and does not require a dedicated port. -The file server works on environments passed to the master, each environment -can have multiple root directories, the subdirectories in the multiple file +The file server works on environments passed to the master. Each environment +can have multiple root directories. The subdirectories in the multiple file roots cannot match, otherwise the downloaded files will not be able to be -reliably ensured. A base environment is required to house the top file +reliably ensured. A base environment is required to house the top file. Example: .. code-block:: yaml @@ -306,12 +306,11 @@ The buffer size in the file server in bytes Syndic Server Settings ---------------------- -The Salt syndic is used to pass commands through a master from a higher -master. Using the syndic is simple, if this is a master that will have -syndic servers(s) below it set the "order_masters" setting to True, if this +A Salt syndic is a Salt master used to pass commands from a higher Salt master to +minions below the syndic. Using the syndic is simple. If this is a master that +will have syndic servers(s) below it, set the "order_masters" setting to True. If this is a master that will be running a syndic daemon for passthrough the "syndic_master" setting needs to be set to the location of the master server -to recieve commands from .. conf_master:: order_masters @@ -320,7 +319,7 @@ to recieve commands from Default: ``False`` -Extra data needs to be sind with publications if the master os controlling a +Extra data needs to be sent with publications if the master is controlling a lower level master via a syndic minion. If this is the case the order_masters value must be set to True @@ -336,7 +335,7 @@ value must be set to True Default: ``None`` If this master will be running a salt-syndic to connect to a higher level -master specify the higher level master with this configuration value +master, specify the higher level master with this configuration value .. code-block:: yaml @@ -358,7 +357,7 @@ compartmentalization of commands based on individual minions. Default: ``{}`` The configuration uses regular expressions to match minions and then a list -of regular expressions to match functions, the following will allow the +of regular expressions to match functions. The following will allow the minion authenticated as foo.example.com to execute functions from the test and pkg modules @@ -377,7 +376,7 @@ This will allow all minions to execute all commands: .*: - .* -This is not recomanded, since it would allow anyone who gets root on any +This is not recommended, since it would allow anyone who gets root on any single minion to instantly have root on all of the minions! Node Groups @@ -434,7 +433,7 @@ One of 'info', 'quiet', 'critical', 'error', 'debug', 'warning'. Default: ``{}`` Logger levels can be used to tweak specific loggers logging levels. -Imagine you want to have the salt library at the 'warning' level, but, you +Imagine you want to have the salt library at the 'warning' level, but you still wish to have 'salt.modules' at the 'debug' level: .. code-block:: yaml diff --git a/doc/ref/states/index.rst b/doc/ref/states/index.rst index 7a122c3eeed7..cd838e2aaff5 100644 --- a/doc/ref/states/index.rst +++ b/doc/ref/states/index.rst @@ -29,10 +29,11 @@ then enforced. Understanding the Salt State System Components ============================================== -The Salt state system is comprised of a number of components, as a user, an +The Salt state system is comprised of a number of components. As a user, an understanding of the sls and renderer systems are needed. But as a developer, -an understanding of salt states, as well as understanding salt states and how -to write the states used by salt. +an understanding of salt states and how to write the states is needed as well. + + Salt SLS System --------------- @@ -80,8 +81,8 @@ users.admin states. The init.sls files are translated to be the state name of the parent directory, so the ``salt/init.sls`` file translates to the salt state. -The plain files are visible to the minions, as well as the state files, in -salt, everything is a file, there is not "magic translation" of files and file +The plain files are visible to the minions, as well as the state files. In +salt, everything is a file; there is no "magic translation" of files and file types. This means that a state file can be distributed to minions just like a plain text or binary file. @@ -91,8 +92,8 @@ SLS Files The Salt state files are simple sets of data. Since the SLS files are just data they can be represented in a number of different ways. The default format is yaml generated from a jinja template. This allows for the states files to have -all the language constructs of Python, and the simplicity of yaml. State files -can then be complicated jinja templates the translate down to yaml, or just +all the language constructs of Python and the simplicity of yaml. State files +can then be complicated jinja templates that translate down to yaml, or just plain and simple yaml files! The State files are constructed data structures in a simple format. The format @@ -169,7 +170,7 @@ Renderer System The Renderer system is a key component to the state system. SLS files are representations of Salt "high data" structures. All Salt cares about when -reading an sls file is the data structure that is produced from file. +reading an sls file is the data structure that is produced from the file. This allows Salt states to be represented by multiple types of files. The Renderer system can be used to allow different formats to be used for sls diff --git a/doc/topics/installation/index.rst b/doc/topics/installation/index.rst index 217bd75970d6..3d588f28a739 100644 --- a/doc/topics/installation/index.rst +++ b/doc/topics/installation/index.rst @@ -34,7 +34,7 @@ Optional Dependencies .. _`Cython`: http://cython.org/ .. _`Jinja2`: http://jinja.pocoo.org/ -Platform-specific instllation instructions +Platform-specific installation instructions ------------------------------------------ .. toctree:: diff --git a/doc/topics/roadmap/index.rst b/doc/topics/roadmap/index.rst index 034d5b949541..81845a9df433 100644 --- a/doc/topics/roadmap/index.rst +++ b/doc/topics/roadmap/index.rst @@ -17,7 +17,7 @@ appear in the web interface. The features listed here are only listed based on major version releases, the plan is to have a clear long term goal but not to overly dictate the flow of development. The project needs to be flexible enough to be able to receive -now features at a moment's notice. This model should spur ideas and make +new features at a moment's notice. This model should spur ideas and make allowing new community developers to join without issue. So just because something is slated for a later reason does not mean that a developer is going to have their code rejected or stalled. @@ -40,7 +40,7 @@ Module Cross Calls Many instances of using functions when we should be using module cross calls exist in the code. Mostly from modules which were written before cross calls were around. The big thing to look for are subprocess calls, since they should -all be running with the cms module. +all be running with the cmd module. State Return Data Cleanup ````````````````````````` diff --git a/doc/topics/tutorials/states_pt1.rst b/doc/topics/tutorials/states_pt1.rst index df9392aa44f0..8fad248a8ed8 100644 --- a/doc/topics/tutorials/states_pt1.rst +++ b/doc/topics/tutorials/states_pt1.rst @@ -94,7 +94,7 @@ The second line, called the :term:`state declaration`, defines which of the Salt States we are using. In this example, we are using the :mod:`pkg state <salt.states.pkg>` to ensure that a given package is installed. -The third line, called the :term:`function declaration` defines which function +The third line, called the :term:`function declaration`, defines which function in the :mod:`pkg state <salt.states.pkg>` module to call. .. admonition:: Renderers @@ -107,7 +107,7 @@ in the :mod:`pkg state <salt.states.pkg>` module to call. Building the expected data structure is the job of Salt :doc:`renderers </ref/renderers/index>` and they are dead-simple to write. - In this tutorial we will be using YAML in Jinja2 templates which is the + In this tutorial we will be using YAML in Jinja2 templates, which is the default format. You can change the default by changing :conf_master:`renderer` in the master configuration file. diff --git a/doc/topics/tutorials/states_pt3.rst b/doc/topics/tutorials/states_pt3.rst index 8b3e7b532254..d12cb9f044aa 100644 --- a/doc/topics/tutorials/states_pt3.rst +++ b/doc/topics/tutorials/states_pt3.rst @@ -66,8 +66,8 @@ called ``grains`` is made available in the template context: Calling Salt modules from templates =================================== -All of the Salt modules loaded by the minion ave available within the -templating system. This allows data to be gathered in real time, on the target +All of the Salt modules loaded by the minion are available within the +templating system. This allows data to be gathered in real time on the target system. It also allows for shell commands to be run easily from within the sls modules. @@ -103,7 +103,7 @@ trees. You have seen an example of how to spread a Salt tree across several files but in order to be able to have :term:`requisite references <requisite reference>` -span multiple files you must use a :term:`include declaration`. For example: +span multiple files you must use an :term:`include declaration`. For example: ``python-libs.sls``: @@ -129,7 +129,7 @@ span multiple files you must use a :term:`include declaration`. For example: :term:`Extend declaration` -------------------------- -You can modify previous declarations by using a :term:`extend declaration`. For +You can modify previous declarations by using an :term:`extend declaration`. For example the following modifies the Apache tree to also restart Apache when the vhosts file is changed: @@ -164,8 +164,8 @@ vhosts file is changed: ------------------------ You can override the :term:`ID declaration` by using a :term:`name -declaration`. For example the previous example is a bit more maintainable if -rewritten as the following: +declaration`. For example, the previous example is a bit more maintainable if +rewritten as follows: ``mywebsite.sls``: @@ -208,7 +208,7 @@ can be rewritten without the loop: Continue learning ================= -The best way to continue learing about Salt States is to read through the +The best way to continue learning about Salt States is to read through the :doc:`reference documentation </ref/states/index>` and to look through examples of existing :term:`state trees <state tree>`. You can find examples in the `salt-states repository`_ and please send a pull-request on GitHub with any From ea83a765a35a2ce6a4dca13a2f77312472313181 Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Thu, 8 Mar 2012 21:08:45 -0700 Subject: [PATCH 293/598] Fix a typo --- doc/ref/configuration/master.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 122a348106d5..1876e3a1c607 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -91,7 +91,7 @@ execution returns and command executions. Default: :file:`/` -The system root direcotry to operate from, change this to make Salt run from +The system root directory to operate from, change this to make Salt run from an alternative root .. code-block:: yaml From 56a835a5c6bcab287f89e8ee3d7df6a3b3af7847 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Thu, 8 Mar 2012 20:26:11 -0800 Subject: [PATCH 294/598] Move utility functions into utils --- salt/__init__.py | 71 +---------------------------------- salt/cli/__init__.py | 9 +++-- salt/utils/process.py | 44 ++++++++++++++++++++++ salt/utils/verify.py | 63 ++++++++++++++++++++++++++++--- tests/integration/__init__.py | 4 +- 5 files changed, 110 insertions(+), 81 deletions(-) create mode 100644 salt/utils/process.py diff --git a/salt/__init__.py b/salt/__init__.py index 9e23e490f76e..fd05304e2211 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -6,86 +6,19 @@ # Import python libs import os import sys -import stat import optparse -import getpass # Import salt libs, the try block bypasses an issue at build time so that c # modules don't cause the build to fail try: import salt.config - import salt.utils.verify + from salt.utils.process import set_pidfile + from salt.utils.verify import check_user, verify_env except ImportError as e: if e.message != 'No module named _msgpack': raise -def set_pidfile(pidfile): - ''' - Save the pidfile - ''' - pdir = os.path.dirname(pidfile) - if not os.path.isdir(pdir): - os.makedirs(pdir) - try: - with open(pidfile, 'w+') as f: - f.write(str(os.getpid())) - except IOError: - pass - - -def verify_env(dirs): - ''' - Verify that the named directories are in place and that the environment - can shake the salt - ''' - for dir_ in dirs: - if not os.path.isdir(dir_): - try: - cumask = os.umask(63) # 077 - os.makedirs(dir_) - os.umask(cumask) - except OSError, e: - sys.stderr.write('Failed to create directory path "{0}" - {1}\n'.format(dir_, e)) - - mode = os.stat(dir_) - # TODO: Should this log if it can't set the permissions - # to very secure for these PKI cert directories? - if not stat.S_IMODE(mode.st_mode) == 448: - if os.access(dir_, os.W_OK): - os.chmod(dir_, 448) - # Run the extra verification checks - salt.utils.verify.run() - - -def check_user(user, log): - ''' - Check user and assign process uid/gid. - ''' - if 'os' in os.environ: - if os.environ['os'].startswith('Windows'): - return True - if user == getpass.getuser(): - return True - import pwd # after confirming not running Windows - try: - p = pwd.getpwnam(user) - try: - os.setgid(p.pw_gid) - os.setuid(p.pw_uid) - except OSError: - if user == 'root': - msg = 'Sorry, the salt must run as root. http://xkcd.com/838' - else: - msg = 'Salt must be run from root or user "{0}"'.format(user) - log.critical(msg) - return False - except KeyError: - msg = 'User not found: "{0}"'.format(user) - log.critical(msg) - return False - return True - class Master(object): ''' Creates a master server diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 70a3ce712b29..c3897d6a4815 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -16,6 +16,7 @@ import salt.output import salt.runner +from salt.utils.verify import verify_env from salt import __version__ as VERSION from salt.exceptions import SaltInvocationError, SaltClientError, \ SaltException @@ -703,10 +704,10 @@ def __parse(self): opts['fun'] = '' opts['arg'] = [] - salt.verify_env([opts['pki_dir'], - opts['cachedir'], - os.path.dirname(opts['log_file']), - ]) + verify_env([opts['pki_dir'], + opts['cachedir'], + os.path.dirname(opts['log_file']), + ]) return opts diff --git a/salt/utils/process.py b/salt/utils/process.py new file mode 100644 index 000000000000..15c87388ede6 --- /dev/null +++ b/salt/utils/process.py @@ -0,0 +1,44 @@ +import logging +import os +import signal +import time + +log = logging.getLogger(__name__) + + +def set_pidfile(pidfile): + ''' + Save the pidfile + ''' + pdir = os.path.dirname(pidfile) + if not os.path.isdir(pdir): + os.makedirs(pdir) + try: + with open(pidfile, 'w+') as f: + f.write(str(os.getpid())) + except IOError: + pass + + +def clean_proc(proc, wait_for_kill=10): + ''' + Generic method for cleaning up multiprocessing procs + ''' + # NoneType and other fun stuff need not apply + if not proc: + return + try: + waited = 0 + while proc.is_alive(): + proc.terminate() + waited += 1 + time.sleep(0.1) + if proc.is_alive() and (waited >= wait_for_kill): + log.error(('Process did not die with terminate(): {0}' + .format(proc.pid))) + os.kill(signal.SIGKILL, proc.pid) + except (AssertionError, AttributeError) as e: + # Catch AssertionError when the proc is evaluated inside the child + # Catch AttributeError when the process dies between proc.is_alive() + # and proc.terminate() and turns into a NoneType + pass diff --git a/salt/utils/verify.py b/salt/utils/verify.py index ddf861e8c0f2..33a898d879fe 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -2,13 +2,17 @@ A few checks to make sure the environment is sane ''' # Original Author: Jeff Schroeder <jeffschroeder@computer.org> - +import getpass +import os +import stat import sys import logging + log = logging.getLogger(__name__) __all__ = ('zmq_version', 'run') + def zmq_version(): '''ZeroMQ python bindings >= 2.1.9 are required''' import zmq @@ -19,8 +23,55 @@ def zmq_version(): return False return True -def run(): - for func in __all__: - if func == "run": continue - if not globals().get(func)(): - continue + +def verify_env(dirs): + ''' + Verify that the named directories are in place and that the environment + can shake the salt + ''' + for dir_ in dirs: + if not os.path.isdir(dir_): + try: + cumask = os.umask(63) # 077 + os.makedirs(dir_) + os.umask(cumask) + except OSError, e: + sys.stderr.write('Failed to create directory path "{0}" - {1}\n'.format(dir_, e)) + + mode = os.stat(dir_) + # TODO: Should this log if it can't set the permissions + # to very secure for these PKI cert directories? + if not stat.S_IMODE(mode.st_mode) == 448: + if os.access(dir_, os.W_OK): + os.chmod(dir_, 448) + # Run the extra verification checks + zmq_version() + + +def check_user(user, log): + ''' + Check user and assign process uid/gid. + ''' + if 'os' in os.environ: + if os.environ['os'].startswith('Windows'): + return True + if user == getpass.getuser(): + return True + import pwd # after confirming not running Windows + try: + p = pwd.getpwnam(user) + try: + os.setgid(p.pw_gid) + os.setuid(p.pw_uid) + except OSError: + if user == 'root': + msg = 'Sorry, the salt must run as root. http://xkcd.com/838' + else: + msg = 'Salt must be run from root or user "{0}"'.format(user) + log.critical(msg) + return False + except KeyError: + msg = 'User not found: "{0}"'.format(user) + log.critical(msg) + return False + return True diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index feac49f35e6c..67b35d59057d 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -6,7 +6,7 @@ import salt.config import salt.master import salt.minion - +from salt.util.verify import verify_env from saltunittest import TestCase INTEGRATION_TEST_DIR = os.path.dirname(os.path.normpath(os.path.abspath(__file__))) @@ -25,7 +25,7 @@ def __enter__(self): ''' self.master_opts = salt.config.master_config(os.path.join(INTEGRATION_TEST_DIR, 'files/conf/master')) self.minion_opts = salt.config.minion_config(os.path.join(INTEGRATION_TEST_DIR, 'files/conf/minion')) - salt.verify_env([os.path.join(self.master_opts['pki_dir'], 'minions'), + verify_env([os.path.join(self.master_opts['pki_dir'], 'minions'), os.path.join(self.master_opts['pki_dir'], 'minions_pre'), os.path.join(self.master_opts['pki_dir'], 'minions_rejected'), os.path.join(self.master_opts['cachedir'], 'jobs'), From b568d1038b4ccf024211f33fe2c9e7483cbabf2d Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Fri, 9 Mar 2012 01:28:31 -0500 Subject: [PATCH 295/598] add the ability to push ssh authorized keys from a file, rather than specifying the full key in the state --- salt/modules/ssh.py | 94 ++++++++++++++++++++++++++++++----------- salt/states/ssh_auth.py | 35 +++++++++++---- 2 files changed, 96 insertions(+), 33 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 4dd661ba3d79..3bb546ae3b67 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -114,30 +114,42 @@ def auth_keys(user, config='.ssh/authorized_keys'): full = os.path.join(uinfo['home'], config) if not os.path.isfile(full): return {} - for line in open(full, 'r').readlines(): - if line.startswith('#'): - # Commented Line - continue - comps = line.split() - if len(comps) < 2: - # Not a valid line - continue - if not comps[0].startswith('ssh-'): - # It has options, grab them - options = comps[0].split(',') - else: - options = [] - if not options: - enc = comps[0] - key = comps[1] - comment = ' '.join(comps[2:]) - else: - enc = comps[1] - key = comps[2] - comment = ' '.join(comps[3:]) - ret[key] = {'enc': enc, - 'comment': comment, - 'options': options} + + return _validate_keys(full) + + +def _validate_keys(key_file): + ''' + Return a dict containing validated keys in the passed file + ''' + ret = {} + try: + for line in open(key_file, 'r').readlines(): + if line.startswith('#'): + # Commented Line + continue + comps = line.split() + if len(comps) < 2: + # Not a valid line + continue + if not comps[0].startswith('ssh-'): + # It has options, grab them + options = comps[0].split(',') + else: + options = [] + if not options: + enc = comps[0] + key = comps[1] + comment = ' '.join(comps[2:]) + else: + enc = comps[1] + key = comps[2] + comment = ' '.join(comps[3:]) + ret[key] = {'enc': enc, + 'comment': comment, + 'options': options} + except IOError: + return "fail" return ret @@ -185,6 +197,38 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): return 'Key removed' return 'Key not present' +def set_auth_key_from_file( + user, + source, + config='.ssh/authorized_keys'): + ''' + Add a key to the authorized_keys file, using a file as the source. + + CLI Example:: + + salt '*' ssh.set_auth_key_from_file <user> salt://ssh_keys/<user>.id_rsa.pub + ''' + # TODO: add support for pulling keys from other file sources as well + lfile = __salt__['cp.cache_file'](source) + if not os.path.isfile(lfile): + return 'fail' + + newkey = {} + rval = "" + newkey = _validate_keys(lfile) + for k in newkey.keys(): + rval += set_auth_key(user, k, newkey[k]['enc'], newkey[k]['comment'], newkey[k]['options'], config) + # Due to the ability for a single file to have multiple keys, it's possible for a single call + # to this function to have both "replace" and "new" as possible valid returns. I ordered the + # following as I thought best. + if 'fail' in rval: + return 'fail' + elif 'replace' in rval: + return 'replace' + elif 'new' in rval: + return 'new' + else: + return 'no change' def set_auth_key( user, @@ -198,7 +242,7 @@ def set_auth_key( CLI Example:: - salt '*' ssh.set_auth_key <user> <key> dsa '[]' .ssh/authorized_keys + salt '*' ssh.set_auth_key <user> <key> dsa 'my key' '[]' .ssh/authorized_keys ''' enc = _refine_enc(enc) replace = False diff --git a/salt/states/ssh_auth.py b/salt/states/ssh_auth.py index 04beba0e2658..3896ce34626f 100644 --- a/salt/states/ssh_auth.py +++ b/salt/states/ssh_auth.py @@ -12,6 +12,12 @@ - present - user: root - enc: ssh-dss + + thatch: + ssh_auth: + - present + - user: root + - source: salt://ssh_keys/thatch.id_rsa.pub ''' @@ -20,6 +26,7 @@ def present( user, enc='ssh-rsa', comment='', + source='', options=[], # FIXME: mutable type; http://goo.gl/ToU2z config='.ssh/authorized_keys'): ''' @@ -37,6 +44,11 @@ def present( comment The comment to be placed with the ssh public key + source + The source file for the key(s). Can contain any number of public keys, + in standard "authorized_keys" format. If this is set, comment, enc, + and options will be ignored. + options The options passed to the key, pass a list object @@ -49,13 +61,20 @@ def present( 'result': True, 'comment': ''} - data = __salt__['ssh.set_auth_key']( - user, - name, - enc, - comment, - options, - config) + if source != '': + data = __salt__['ssh.set_auth_key_from_file']( + user, + source, + config) + else: + data = __salt__['ssh.set_auth_key']( + user, + name, + enc, + comment, + source, + options, + config) if data == 'replace': ret['changes'][name] = 'Updated' @@ -72,7 +91,7 @@ def present( elif data == 'fail': ret['result'] = False ret['comment'] = ('Failed to add the ssh key, is the home directory' - ' available?') + ' available and/or does the key file exist?') return ret From 6f5b6a63f70122d9430c4aad27f20d7f42f7ad54 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Thu, 8 Mar 2012 23:47:34 -0800 Subject: [PATCH 296/598] Fix integration tests on osx --- salt/grains/core.py | 1 + tests/integration/__init__.py | 12 +++++++++--- tests/integration/files/conf/master | 12 ++++++------ tests/integration/files/conf/minion | 8 ++++---- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 18a965bcfd98..b6f2c655790a 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -425,6 +425,7 @@ def os_data(): grains['os'] = 'ESXi' elif grains['kernel'] == 'Darwin': grains['os'] = 'MacOS' + grains.update(_freebsd_cpudata()) else: grains['os'] = grains['kernel'] if grains['kernel'] == 'Linux': diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 67b35d59057d..1b05d6870b15 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -6,7 +6,7 @@ import salt.config import salt.master import salt.minion -from salt.util.verify import verify_env +from salt.utils.verify import verify_env from saltunittest import TestCase INTEGRATION_TEST_DIR = os.path.dirname(os.path.normpath(os.path.abspath(__file__))) @@ -23,8 +23,14 @@ def __enter__(self): ''' Start a master and minion ''' - self.master_opts = salt.config.master_config(os.path.join(INTEGRATION_TEST_DIR, 'files/conf/master')) - self.minion_opts = salt.config.minion_config(os.path.join(INTEGRATION_TEST_DIR, 'files/conf/minion')) + self.master_opts = salt.config.master_config( + os.path.join(INTEGRATION_TEST_DIR, 'files/conf/master')) + self.minion_opts = salt.config.minion_config( + os.path.join(INTEGRATION_TEST_DIR, 'files/conf/minion')) + self.master_opts['file_roots'] = FILES + self.master_opts['hosts.file'] = os.path.join(TMP, 'hosts') + self.minion_opts['file_roots'] = FILES + self.minion_opts['hosts.file'] = os.path.join(TMP, 'hosts') verify_env([os.path.join(self.master_opts['pki_dir'], 'minions'), os.path.join(self.master_opts['pki_dir'], 'minions_pre'), os.path.join(self.master_opts['pki_dir'], 'minions_rejected'), diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 3fec9b80a4c4..96c07ded80c9 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -1,10 +1,10 @@ -publish_port: 94505 -ret_port: 94506 +publish_port: 64505 +ret_port: 64506 worker_threads: 1 root_dir: /tmp/salttest -pidfile: /tmp/salttest/masterpid -pki_dir: /tmp/salttest/pki -cachedir: /tmp/salttest/cache +pidfile: masterpid +pki_dir: pki +cachedir: cache timeout: 1 -sock_dir: /tmp/salttest/.salt-unix +sock_dir: .salt-unix open_mode: True diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index a3a745216cf6..9c34a05ce15a 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -1,13 +1,13 @@ # basic config master: localhost -master_port: 94506 +master_port: 64506 root_dir: /tmp/salttest -pki_dir: /tmp/salttest/pki +pki_dir: pki id: minion -cachedir: /tmp/salttest/cachedir +cachedir: cachedir acceptance_wait_time: = 1 open_mode: True -log_file: /tmp/salttest/minion +log_file: minion # module extension test.foo: baz From a8ffa3b374dd59c5f6d8619d4a283c10f7a573c9 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 9 Mar 2012 00:36:52 -0800 Subject: [PATCH 297/598] Tuples, how do they work? Report error's correctly for issue #832 --- salt/states/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index 89db5b5af96b..5a92b1188506 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -206,7 +206,7 @@ def _clean_dir(root, keep): def _error(ret, err_msg): - ret['result'] = False, + ret['result'] = False ret['comment'] = err_msg return ret From 8f8d0222b06b09f4949fa823d171a828054c38a3 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 9 Mar 2012 11:16:17 -0800 Subject: [PATCH 298/598] Raise CommandNotFound in virtualenv module --- salt/modules/virtualenv.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/modules/virtualenv.py b/salt/modules/virtualenv.py index 3762a5e83871..4ee6502e39cb 100644 --- a/salt/modules/virtualenv.py +++ b/salt/modules/virtualenv.py @@ -1,12 +1,15 @@ ''' Create virtualenv environments ''' +from salt.exceptions import CommandNotFoundError import salt.utils + __opts__ = { 'venv_bin': 'virtualenv', } + def __virtual__(): ''' Only load the module if virtualenv is installed @@ -14,6 +17,7 @@ def __virtual__(): cmd = __opts__.get('venv_bin', 'virtualenv') return 'virtualenv' if salt.utils.which(cmd) else False + def create(path, venv_bin='', no_site_packages=False, @@ -50,6 +54,10 @@ def create(path, salt '*' pip.virtualenv /path/to/new/virtualenv ''' + if not salt.utils.which(venv_bin): + raise CommandNotFoundError( + "Please install {venv_bin}".format(venv_bin=venv_bin)) + cmd = '{venv_bin} {args} {path}'.format( venv_bin=venv_bin if venv_bin else __opts__['venv_bin'], args=''.join([ From 40fc29b416ba89c2fc850152c2437214a57d4b42 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 9 Mar 2012 11:49:20 -0800 Subject: [PATCH 299/598] Add default for timeout in cli parser --- salt/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index c3897d6a4815..508d2e233aed 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -41,7 +41,7 @@ def __parse(self): parser.add_option('-t', '--timeout', - default=None, + default=5, dest='timeout', help=('Set the return timeout for batch jobs; ' 'default=5 seconds')) From 0e86b4179ffc2fdf1622d0fe3bdff7909a83681e Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 9 Mar 2012 12:09:17 -0800 Subject: [PATCH 300/598] Remove __virtual__ in virtualenv, fix default args to have sane default --- salt/modules/virtualenv.py | 15 +++------------ salt/states/virtualenv.py | 2 +- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/salt/modules/virtualenv.py b/salt/modules/virtualenv.py index 4ee6502e39cb..938fa89ae242 100644 --- a/salt/modules/virtualenv.py +++ b/salt/modules/virtualenv.py @@ -2,7 +2,6 @@ Create virtualenv environments ''' from salt.exceptions import CommandNotFoundError -import salt.utils __opts__ = { @@ -10,16 +9,8 @@ } -def __virtual__(): - ''' - Only load the module if virtualenv is installed - ''' - cmd = __opts__.get('venv_bin', 'virtualenv') - return 'virtualenv' if salt.utils.which(cmd) else False - - def create(path, - venv_bin='', + venv_bin=__opts__['venv_bin'], no_site_packages=False, system_site_packages=False, clear=False, @@ -54,12 +45,12 @@ def create(path, salt '*' pip.virtualenv /path/to/new/virtualenv ''' - if not salt.utils.which(venv_bin): + if not __salt__['cmd.has_exec'](venv_bin): raise CommandNotFoundError( "Please install {venv_bin}".format(venv_bin=venv_bin)) cmd = '{venv_bin} {args} {path}'.format( - venv_bin=venv_bin if venv_bin else __opts__['venv_bin'], + venv_bin=venv_bin, args=''.join([ ' --no-site-packages' if no_site_packages else '', ' --system-site-packages' if system_site_packages else '', diff --git a/salt/states/virtualenv.py b/salt/states/virtualenv.py index 3d952ad48248..c4566feb9b61 100644 --- a/salt/states/virtualenv.py +++ b/salt/states/virtualenv.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def manage(name, - venv_bin='', + venv_bin='virtualenv', requirements='', no_site_packages=False, system_site_packages=False, From 9198c1da9a0dd93427fac1a79a0dc766dadf61fc Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 9 Mar 2012 12:42:11 -0800 Subject: [PATCH 301/598] Bail out early from SaltCMD.run if the local client cannot be configured --- salt/cli/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 508d2e233aed..6815d6616b54 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -41,7 +41,7 @@ def __parse(self): parser.add_option('-t', '--timeout', - default=5, + default=None, dest='timeout', help=('Set the return timeout for batch jobs; ' 'default=5 seconds')) @@ -236,6 +236,9 @@ def run(self): local = None ret = exc out = '' + self._output_ret(ret, out) + return + if 'query' in self.opts: ret = local.find_cmd(self.opts['cmd']) for jid in ret: From 58b9e89a31bb43239b0278ab043f52ca9f543a99 Mon Sep 17 00:00:00 2001 From: archme <archme.mail@gmail.com> Date: Fri, 9 Mar 2012 22:38:40 +0100 Subject: [PATCH 302/598] Fix network.netstat incompatibility with "netstat 1.42 (2001-04-15) from net-tools 1.6.0" --- salt/modules/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/network.py b/salt/modules/network.py index f2c9f6aa2ece..2065564a05af 100644 --- a/salt/modules/network.py +++ b/salt/modules/network.py @@ -56,7 +56,7 @@ def netstat(): ''' ret = [] cmd = 'netstat -tulpnea' - out = __salt__['cmd.run'](cmd) + out = __salt__['cmd.run'](cmd).split('\n') for line in out: comps = line.split() if line.startswith('tcp'): From 0a90eee12873b0641ec6fd90e944f54cf317bc9f Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 9 Mar 2012 16:04:04 -0800 Subject: [PATCH 303/598] Add warning about default shell in Popen, #870 --- salt/states/cmd.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/states/cmd.py b/salt/states/cmd.py index f3ecd7d53b80..d984c1a11af2 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -31,6 +31,11 @@ - run - unless: echo 'foo' > /tmp/.test +.. warning:: + + Both ``onlyif`` and ``unless`` are using ``/bin/sh`` as their shell for + execution + ''' import grp From bff7483459c1b10e328828353bc546328374d070 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Fri, 9 Mar 2012 22:04:08 -0800 Subject: [PATCH 304/598] Import the exceptions that are raised --- salt/utils/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index c14beefad2af..0fcf0cda9122 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -8,6 +8,8 @@ import socket import sys +from salt.exceptions import SaltClientError + log = logging.getLogger(__name__) From 9aac5b916a4bba6513d09b1e1c03ad09ed0d297a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 10 Mar 2012 00:18:28 -0700 Subject: [PATCH 305/598] More verbose warning, fixes #870 --- salt/states/cmd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/states/cmd.py b/salt/states/cmd.py index d984c1a11af2..9f91629280d6 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -33,8 +33,10 @@ .. warning:: - Both ``onlyif`` and ``unless`` are using ``/bin/sh`` as their shell for - execution + Please be advised that on Unix systems the shell being used by python + to run executions is /bin/sh, this requires that commands are formatted + to execute under /bin/sh. Some capabilities of newer shells such as bash, + zsh and ksh will not always be available on minions. ''' From d91255a7cd4d860e4e12b9caa154d98d09c26d4c Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Fri, 9 Mar 2012 22:14:50 -0700 Subject: [PATCH 306/598] Remove -E option from call to pip This option was removed in recent pip versions. When installing packages to a virtualenv it is recommended to just call out to the pip inside the virtualenv. --- salt/modules/pip.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index 3dddce3acd3d..43e7f7f22be3 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -42,9 +42,8 @@ def install(env='', requirements='', pkgs='', pip_bin=''): salt '*' pip.install /var/www/myvirtualenv.com \\ /path/to/requirements.txt ''' - cmd = '{pip_bin} install {env} {reqs} {pkgs}'.format( + cmd = '{pip_bin} install {reqs} {pkgs}'.format( pip_bin=_get_pip_bin(pip_bin, env), - env='-E {0}'.format(env) if env else '', reqs='-r {0}'.format(requirements) if requirements else '', pkgs=pkgs) @@ -64,8 +63,6 @@ def freeze(env='', pip_bin=''): to the pip that is installed in the virtualenv. This option can also be set in the minion config file as ``pip.pip_bin``. ''' - # Using freeze with -E seems to be twitchy on older pips so call the pip - # inside the venv if using a venv cmd = '{0} freeze'.format(_get_pip_bin(pip_bin, env)) return __salt__['cmd.run'](cmd).split('\n') From 1892fc2bb98ca3067afbc93b26da2a1068cd137c Mon Sep 17 00:00:00 2001 From: Frank Klaassen <frank@netvisions.nl> Date: Sat, 10 Mar 2012 12:12:20 +0100 Subject: [PATCH 307/598] Fixed arguments for ssh module (needs 6 args, not 7) after the change in b568d103 --- salt/states/ssh_auth.py | 1 - 1 file changed, 1 deletion(-) mode change 100644 => 100755 salt/states/ssh_auth.py diff --git a/salt/states/ssh_auth.py b/salt/states/ssh_auth.py old mode 100644 new mode 100755 index 3896ce34626f..d769e0a4139c --- a/salt/states/ssh_auth.py +++ b/salt/states/ssh_auth.py @@ -72,7 +72,6 @@ def present( name, enc, comment, - source, options, config) From 7512c7fb47cb11d6cd1e69defa3a41d04d4afceb Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 10 Mar 2012 07:27:38 -0800 Subject: [PATCH 308/598] Test the corresponding cmd function --- tests/integration/modules/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/modules/test.py b/tests/integration/modules/test.py index f9c0856a81ec..8b830d4e79df 100644 --- a/tests/integration/modules/test.py +++ b/tests/integration/modules/test.py @@ -78,10 +78,10 @@ def test_collatz(self): ''' self.assertEqual( self.run_function( - 'test.fib', + 'test.collatz', ['40'], )[0][-1], - 34 + 2 ) def test_outputter(self): From 7fa999fd683e6c43e8f42f67d5892f895b841305 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 10 Mar 2012 07:42:18 -0800 Subject: [PATCH 309/598] Add kwargs support to remote commands, #686 --- salt/cli/caller.py | 7 ++++--- salt/minion.py | 9 ++++++--- salt/state.py | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/salt/cli/caller.py b/salt/cli/caller.py index e792cc8cbfa6..03725af044c6 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -11,6 +11,7 @@ import salt.utils import salt.loader import salt.minion +import salt.state # Custom exceptions from salt.exceptions import CommandExecutionError, CommandNotFoundError @@ -38,9 +39,9 @@ def call(self): sys.stderr.write('Function {0} is not available\n'.format(fun)) sys.exit(1) try: - ret['return'] = self.minion.functions[fun]( - *self.opts['arg'] - ) + args, kw = salt.state.build_args( + self.minion.functions[fun], self.opts['arg']) + ret['return'] = self.minion.functions[fun](*args, **kw) except (TypeError, CommandExecutionError) as exc: msg = 'Error running \'{0}\': {1}\n' sys.stderr.write(msg.format(fun, str(exc))) diff --git a/salt/minion.py b/salt/minion.py index 7741b51fbd43..148c9c5f48c5 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -236,7 +236,9 @@ def _thread_return(self, data): function_name = data['fun'] if function_name in self.functions: try: - ret['return'] = self.functions[data['fun']](*data['arg']) + func = self.functions[data['fun']] + args, kw = salt.state.build_args(func, data['arg']) + ret['return'] = func(*args, **kw) except CommandNotFoundError as exc: msg = 'Command not found in \'{0}\': {1}' log.debug(msg.format(function_name, str(exc))) @@ -293,8 +295,9 @@ def _thread_multi_return(self, data): pass try: - ret['return'][data['fun'][ind]]\ - = self.functions[data['fun'][ind]](*data['arg'][ind]) + func = self.functions[data['fun'][ind]] + args, kw = salt.state.build_args(func, data['arg'][ind]) + ret['return'][data['fun'][ind]] = func(*args, **kw) except Exception as exc: trb = traceback.format_exc() log.warning( diff --git a/salt/state.py b/salt/state.py index 760c516a6bfa..9a37c003fa38 100644 --- a/salt/state.py +++ b/salt/state.py @@ -59,6 +59,26 @@ def _getargs(func): return aspec +def build_args(func, args): + spec_args, _, _, _ = _getargs(func) + + _args = [] + _kw = {} + for arg in args: + if isinstance(arg, basestring): + arg = arg.split('=', 1) + arg_len = len(arg) + if arg_len == 2: + k, v = arg + if k in spec_args: + _kw[k] = v + else: + _args.append(arg[0]) + else: + _args.append(arg) + return _args, _kw + + def format_log(ret): ''' Format the state into a log message From 3db4ada07477fb5a13a91999b6d28f5993bdbc05 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 10 Mar 2012 21:11:50 -0800 Subject: [PATCH 310/598] Validate for bad requisite blocks, issue #880 --- salt/state.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/salt/state.py b/salt/state.py index 9a37c003fa38..4b0a8a9f98c2 100644 --- a/salt/state.py +++ b/salt/state.py @@ -117,6 +117,14 @@ def format_log(ret): log.info(str(ret)) +def ishashable(obj): + try: + hash(obj) + except TypeError: + return False + return True + + class StateError(Exception): ''' Custom exception class. @@ -345,6 +353,12 @@ def verify_high(self, high): req, body['__sls__']) errors.append(err) + req_key = req.keys()[0] + req_val = req[req_key] + if not ishashable(req_val): + errors.append(( + 'Illegal requisite "{0}", please check your syntax.\n' + ).format(str(req_val))) # Make sure that there is only one key in the dict if len(arg.keys()) != 1: errors.append(('Multiple dictionaries defined' @@ -622,9 +636,11 @@ def check_requisite(self, low, running, chunks): for req in low[r_state]: found = False for chunk in chunks: - if fnmatch.fnmatch(chunk['__id__'], req[req.keys()[0]]) or \ - fnmatch.fnmatch(chunk['name'], req[req.keys()[0]]): - if chunk['state'] == req.keys()[0]: + req_key = req.keys()[0] + req_val = req[req_key] + if (fnmatch.fnmatch(chunk['name'], req_val) or + fnmatch.fnmatch(chunk['__id__'], req_val)): + if chunk['state'] == req_key: found = True reqs[r_state].append(chunk) if not found: @@ -672,9 +688,10 @@ def call_chunk(self, low, running, chunks): for req in low[requisite]: found = False for chunk in chunks: - if fnmatch.fnmatch(chunk['name'], req[req.keys()[0]]) \ - or fnmatch.fnmatch(chunk['__id__'], - req[req.keys()[0]]): + req_key = req.keys()[0] + req_val = req[req_key] + if (fnmatch.fnmatch(chunk['name'], req_val) or + fnmatch.fnmatch(chunk['__id__'], req_val)): if chunk['state'] == req.keys()[0]: reqs.append(chunk) found = True From fecb9709142b95ae9f07b7d597b1e98f77ebc954 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 10 Mar 2012 21:13:19 -0800 Subject: [PATCH 311/598] Display tracebacks from bad command execution in salt-call --- salt/cli/caller.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/cli/caller.py b/salt/cli/caller.py index 03725af044c6..22299d5f8a4e 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -5,6 +5,7 @@ # Import python modules import sys +import traceback # Import salt libs import salt @@ -44,6 +45,7 @@ def call(self): ret['return'] = self.minion.functions[fun](*args, **kw) except (TypeError, CommandExecutionError) as exc: msg = 'Error running \'{0}\': {1}\n' + sys.stderr.write(traceback.format_exc()) sys.stderr.write(msg.format(fun, str(exc))) sys.exit(1) except CommandNotFoundError as exc: From d22b85ea80bf596a3112cf165291cd45e7a82bd5 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 10 Mar 2012 21:39:22 -0800 Subject: [PATCH 312/598] Allow run to define a shell, use /bin/sh as default --- salt/modules/cmd.py | 55 ++++++++++++++++++++++++++------------------- salt/states/cmd.py | 5 +++-- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/salt/modules/cmd.py b/salt/modules/cmd.py index f79c4ea07703..031925ac7ed3 100644 --- a/salt/modules/cmd.py +++ b/salt/modules/cmd.py @@ -24,13 +24,19 @@ __outputter__ = { 'run': 'txt', } + + +DEFAULT_SHELL = '/bin/sh' + + def _run(cmd, - cwd=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - quiet=False, - runas=None, - with_env=True): + cwd=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + quiet=False, + runas=None, + with_env=True, + shell=DEFAULT_SHELL): ''' Do the DRY thing and only call subprocess.Popen() once ''' @@ -78,11 +84,12 @@ def _run(cmd, # This is where the magic happens proc = subprocess.Popen(cmd, - cwd=cwd, - shell=True, - stdout=stdout, - stderr=stderr - ) + executable=shell, + cwd=cwd, + shell=True, + stdout=stdout, + stderr=stderr + ) out = proc.communicate() ret['stdout'] = out[0] @@ -92,14 +99,15 @@ def _run(cmd, return ret -def _run_quiet(cmd, cwd=None, runas=None): +def _run_quiet(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): ''' Helper for running commands quietly for minion startup ''' - return _run(cmd, runas=runas, cwd=cwd, stderr=subprocess.STDOUT, quiet=True)['stdout'] + return _run(cmd, runas=runas, cwd=cwd, stderr=subprocess.STDOUT, + quiet=True, shell=shell)['stdout'] -def run(cmd, cwd=None, runas=None): +def run(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): ''' Execute the passed command and return the output as a string @@ -107,12 +115,13 @@ def run(cmd, cwd=None, runas=None): salt '*' cmd.run "ls -l | awk '/foo/{print $2}'" ''' - out = _run(cmd, runas=runas, cwd=cwd, stderr=subprocess.STDOUT)['stdout'] + out = _run(cmd, runas=runas, shell=shell, + cwd=cwd, stderr=subprocess.STDOUT)['stdout'] log.debug(out) return out -def run_stdout(cmd, cwd=None, runas=None): +def run_stdout(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): ''' Execute a command, and only return the standard out @@ -120,12 +129,12 @@ def run_stdout(cmd, cwd=None, runas=None): salt '*' cmd.run_stdout "ls -l | awk '/foo/{print $2}'" ''' - stdout = _run(cmd, runas=runas, cwd=cwd)["stdout"] + stdout = _run(cmd, runas=runas, cwd=cwd, shell=shell)["stdout"] log.debug(stdout) return stdout -def run_stderr(cmd, cwd=None, runas=None): +def run_stderr(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): ''' Execute a command and only return the standard error @@ -133,12 +142,12 @@ def run_stderr(cmd, cwd=None, runas=None): salt '*' cmd.run_stderr "ls -l | awk '/foo/{print $2}'" ''' - stderr = _run(cmd, runas=runas, cwd=cwd)["stderr"] + stderr = _run(cmd, runas=runas, cwd=cwd, shell=shell)["stderr"] log.debug(stderr) return stderr -def run_all(cmd, cwd=None, runas=None): +def run_all(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): ''' Execute the passed command and return a dict of return data @@ -146,7 +155,7 @@ def run_all(cmd, cwd=None, runas=None): salt '*' cmd.run_all "ls -l | awk '/foo/{print $2}'" ''' - ret = _run(cmd, runas=runas, cwd=cwd) + ret = _run(cmd, runas=runas, cwd=cwd, shell=shell) if ret['retcode'] != 0: log.error('Command {0} failed'.format(cmd)) log.error('retcode: {0}'.format(ret['retcode'])) @@ -158,7 +167,7 @@ def run_all(cmd, cwd=None, runas=None): return ret -def retcode(cmd, cwd=None, runas=None): +def retcode(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): ''' Execute a shell command and return the command's return code. @@ -166,7 +175,7 @@ def retcode(cmd, cwd=None, runas=None): salt '*' cmd.retcode "file /bin/bash" ''' - return _run(cmd, runas=runas, cwd=cwd)['retcode'] + return _run(cmd, runas=runas, cwd=cwd, shell=shell)['retcode'] def has_exec(cmd): diff --git a/salt/states/cmd.py b/salt/states/cmd.py index 9f91629280d6..6d1318ccceaa 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -86,7 +86,8 @@ def run(name, unless=None, cwd='/root', user=None, - group=None): + group=None, + shell='/bin/sh'): ''' Run a command if certain circumstances are met @@ -145,7 +146,7 @@ def run(name, # Wow, we passed the test, run this sucker! try: - cmd_all = __salt__['cmd.run_all'](name, cwd, runas=user) + cmd_all = __salt__['cmd.run_all'](name, cwd, runas=user, shell=shell) except CommandExecutionError as e: ret['comment'] = e return ret From 751449ab8522c525d1ee64ec58d737d5204bacc6 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 10 Mar 2012 23:07:06 -0800 Subject: [PATCH 313/598] Add env to the cmd state and module, #789 --- salt/modules/cmd.py | 30 +++++++++++++++++------------- salt/states/cmd.py | 17 +++++++++++++++-- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/salt/modules/cmd.py b/salt/modules/cmd.py index 031925ac7ed3..fe668e100528 100644 --- a/salt/modules/cmd.py +++ b/salt/modules/cmd.py @@ -36,7 +36,8 @@ def _run(cmd, quiet=False, runas=None, with_env=True, - shell=DEFAULT_SHELL): + shell=DEFAULT_SHELL, + env=()): ''' Do the DRY thing and only call subprocess.Popen() once ''' @@ -82,11 +83,14 @@ def _run(cmd, log.info('Executing command {0} as user {1} in directory {2}'.format( orig_cmd, runas, cwd)) + run_env = os.environ + run_env.update(env) # This is where the magic happens proc = subprocess.Popen(cmd, executable=shell, cwd=cwd, shell=True, + env=run_env, stdout=stdout, stderr=stderr ) @@ -99,15 +103,15 @@ def _run(cmd, return ret -def _run_quiet(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): +def _run_quiet(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL, env=()): ''' Helper for running commands quietly for minion startup ''' return _run(cmd, runas=runas, cwd=cwd, stderr=subprocess.STDOUT, - quiet=True, shell=shell)['stdout'] + quiet=True, shell=shell, env=env)['stdout'] -def run(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): +def run(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL, env=()): ''' Execute the passed command and return the output as a string @@ -116,12 +120,12 @@ def run(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): salt '*' cmd.run "ls -l | awk '/foo/{print $2}'" ''' out = _run(cmd, runas=runas, shell=shell, - cwd=cwd, stderr=subprocess.STDOUT)['stdout'] + cwd=cwd, stderr=subprocess.STDOUT, env=env)['stdout'] log.debug(out) return out -def run_stdout(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): +def run_stdout(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL, env=()): ''' Execute a command, and only return the standard out @@ -129,12 +133,12 @@ def run_stdout(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): salt '*' cmd.run_stdout "ls -l | awk '/foo/{print $2}'" ''' - stdout = _run(cmd, runas=runas, cwd=cwd, shell=shell)["stdout"] + stdout = _run(cmd, runas=runas, cwd=cwd, shell=shell, env=())["stdout"] log.debug(stdout) return stdout -def run_stderr(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): +def run_stderr(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL, env=()): ''' Execute a command and only return the standard error @@ -142,12 +146,12 @@ def run_stderr(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): salt '*' cmd.run_stderr "ls -l | awk '/foo/{print $2}'" ''' - stderr = _run(cmd, runas=runas, cwd=cwd, shell=shell)["stderr"] + stderr = _run(cmd, runas=runas, cwd=cwd, shell=shell, env=env)["stderr"] log.debug(stderr) return stderr -def run_all(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): +def run_all(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL, env=()): ''' Execute the passed command and return a dict of return data @@ -155,7 +159,7 @@ def run_all(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): salt '*' cmd.run_all "ls -l | awk '/foo/{print $2}'" ''' - ret = _run(cmd, runas=runas, cwd=cwd, shell=shell) + ret = _run(cmd, runas=runas, cwd=cwd, shell=shell, env=env) if ret['retcode'] != 0: log.error('Command {0} failed'.format(cmd)) log.error('retcode: {0}'.format(ret['retcode'])) @@ -167,7 +171,7 @@ def run_all(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): return ret -def retcode(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): +def retcode(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL, env=()): ''' Execute a shell command and return the command's return code. @@ -175,7 +179,7 @@ def retcode(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL): salt '*' cmd.retcode "file /bin/bash" ''' - return _run(cmd, runas=runas, cwd=cwd, shell=shell)['retcode'] + return _run(cmd, runas=runas, cwd=cwd, shell=shell, env=env)['retcode'] def has_exec(cmd): diff --git a/salt/states/cmd.py b/salt/states/cmd.py index 6d1318ccceaa..c900b7e55079 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -87,7 +87,8 @@ def run(name, cwd='/root', user=None, group=None, - shell='/bin/sh'): + shell='/bin/sh', + env=()): ''' Run a command if certain circumstances are met @@ -144,9 +145,21 @@ def run(name, ret['comment'] = 'The group ' + group + ' is not available' return ret + if env: + _env = {} + for var in env.split(): + try: + k, v = var.split('=') + _env[k] = v + except ValueError: + ret['comment'] = 'Invalid enviromental var: "{0}"' % var + return ret + env = _env + # Wow, we passed the test, run this sucker! try: - cmd_all = __salt__['cmd.run_all'](name, cwd, runas=user, shell=shell) + cmd_all = __salt__['cmd.run_all'](name, cwd, runas=user, + shell=shell, env=env) except CommandExecutionError as e: ret['comment'] = e return ret From 94e8eab349a53e385f7540863cc22ed08ad8ee1a Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Tue, 22 Nov 2011 15:30:01 -0700 Subject: [PATCH 314/598] Added modules for working with Git and Mercurial repositories --- salt/modules/git.py | 145 ++++++++++++++++++++++++++++++++++++++++++++ salt/modules/hg.py | 87 ++++++++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 salt/modules/git.py create mode 100644 salt/modules/hg.py diff --git a/salt/modules/git.py b/salt/modules/git.py new file mode 100644 index 000000000000..949f4fdbc1e4 --- /dev/null +++ b/salt/modules/git.py @@ -0,0 +1,145 @@ +''' +Support for the Git SCM +''' +import os + +def _git_getdir(cwd): + ''' + Returns the absolute path to the top-level of a given repo because some Git + commands are sensitive to where they're run from (archive for one) + ''' + cmd_bare = 'git rev-parse --is-bare-repository' + is_bare = __salt__['cmd.run_stdout'](cmd_bare, cwd) == 'true' + + if is_bare: + return cwd + + cmd_toplvl = 'git rev-parse --show-toplevel' + return __salt__['cmd.run'](cmd_toplvl, cwd) + +def revision(cwd, rev='HEAD', short=False): + ''' + Returns the long hash of a given identifier (hash, branch, tag, HEAD, etc) + + Usage:: + + salt '*' git.revision /path/to/repo mybranch + ''' + cmd = 'git rev-parse {0}{1}'.format('--short ' if short else '', rev) + result = __salt__['cmd.run_all'](cmd, cwd) + + if result['retcode'] == 0: + return result['stdout'].strip('\n') + else: + return '' + +def clone(cwd, repository, opts=''): + ''' + Clone a new repository + + Usage:: + + salt '*' git.clone /path/to/repo git://github.com/saltstack/salt.git + + salt '*' git.clone /path/to/repo.git\\ + git://github.com/saltstack/salt.git '--bare --origin github' + + ''' + cmd = 'git clone {0} {1} {2}'.format(repository, cwd, opts) + return __salt__['cmd.run'](cmd) + +def describe(cwd, rev='HEAD'): + ''' + Returns the git describe string (or the SHA hash if there are no tags) for + the given revision + ''' + cmd = 'git describe {0}'.format(rev) + return __salt__['cmd.run_stdout'](cmd, cwd=cwd).strip('\n') + +def archive(cwd, output, rev='HEAD', fmt='', prefix=''): + ''' + Export a tarball from the repository + + If ``prefix`` is not specified it defaults to the basename of the repo + directory. + + Usage:: + + salt '*' git.archive /path/to/repo /path/to/archive.tar.gz + ''' + basename = '{0}/'.format(os.path.basename(_git_getdir(cwd)).strip('\n')) + + cmd = 'git archive{prefix}{fmt} -o {output} {rev}'.format( + rev = rev, + output = output, + fmt = ' --format={0}'.format(fmt) if fmt else '', + prefix = ' --prefix="{0}"'.format(prefix if prefix else basename)) + + return __salt__['cmd.run'](cmd, cwd=cwd) + +def fetch(cwd, opts=''): + ''' + Perform a fetch on the given repository + + Usage:: + + salt '*' git.fetch /path/to/repo '--all' + ''' + return __salt__['cmd.run']('git fetch {0}'.format(opts), cwd=cwd) + +def pull(cwd, opts=''): + ''' + Perform a pull on the given repository + + Usage:: + + salt '*' git.pull /path/to/repo '--rebase origin master' + ''' + return __salt__['cmd.run']('git pull {0}'.format(opts), cwd=cwd) + +def checkout(cwd, rev, force=False, opts=''): + ''' + Checkout a given revision + + Usage:: + + salt '*' git.checkout /path/to/repo somebranch + + salt '*' git.checkout /path/to/repo 'testbranch -- conf/file1 file2' + + salt '*' git.checkout /path/to/repo 'origin/mybranch --track' + ''' + cmd = 'git checkout{0} {1} {2}'.format(' -f' if force else '', rev, opts) + return __salt__['cmd.run'](cmd, cwd=cwd) + +def merge(cwd, branch='@{upstream}', opts=''): + ''' + Merge a given branch + + cwd + The path to the Git repository + branch : @{upstream} + The remote branch or revision to merge into the current branch + opts : (none) + Any additional options to add to the command line + + Usage:: + + salt '*' git.fetch /path/to/repo + salt '*' git.merge /path/to/repo @{upstream} + ''' + cmd = 'git merge {0}{1} {2}'.format( + branch, + opts) + return __salt__['cmd.run'](cmd, cwd).strip('\n') + +def init(cwd, opts=''): + ''' + Init a new repository + + Usage:: + + salt '*' git.init /path/to/repo.git '--bare' + ''' + cmd = 'git init {0} {1}'.format(cwd, opts) + return __salt__['cmd.run'](cmd).strip('\n') diff --git a/salt/modules/hg.py b/salt/modules/hg.py new file mode 100644 index 000000000000..c785059f0762 --- /dev/null +++ b/salt/modules/hg.py @@ -0,0 +1,87 @@ +''' +Support for the Mercurial SCM +''' + +def revision(cwd, rev='tip', short=False): + ''' + Returns the long hash of a given identifier (hash, branch, tag, HEAD, etc) + + Usage:: + + salt '*' hg.revision /path/to/repo mybranch + ''' + cmd = 'hg id -i{short} {ref}'.format( + short = ' --debug' if not short else '', + rev = ' -r {0}'.format(rev)) + + result = __salt__['cmd.run_all'](cmd, cwd=cwd) + + if result['retcode'] == 0: + return result['stdout'].strip('\n') + else: + return '' + +def describe(cwd, rev='tip'): + ''' + Mimick git describe and return an identifier for the given revision + + Usage:: + + salt '*' hg.describe /path/to/repo + ''' + cmd = "hg log -r {0} --template"\ + " '{{latesttag}}-{{latesttagdistance}}-{{node|short}}'".format(rev) + desc = __salt__['cmd.run_stdout'](cmd, cwd=cwd) + + return desc or revision(cwd, rev, short=True) + +def archive(cwd, output, rev='tip', fmt='', prefix=''): + ''' + Export a tarball from the repository + + If ``prefix`` is not specified it defaults to the basename of the repo + directory. + + Usage:: + + salt '*' hg.archive /path/to/repo /path/to/archive.tar.gz + ''' + cmd = 'hg archive {output}{rev}{fmt}'.format( + rev = ' --rev {0}'.format(rev), + output = output, + fmt = ' --type {0}'.format(fmt) if fmt else '', + prefix = ' --prefix "{0}"'.format(prefix if prefix else '')) + + return __salt__['cmd.run'](cmd, cwd=cwd) + +def pull(cwd, opts=''): + ''' + Perform a pull on the given repository + + Usage:: + + salt '*' hg.pull /path/to/repo '-u' + ''' + return __salt__['cmd.run']('hg pull {0}'.format(opts), cwd=cwd) + +def update(cwd, rev, force=False): + ''' + Checkout a given revision + + Usage:: + + salt '*' hg.update /path/to/repo somebranch + ''' + cmd = 'hg update {0}{1}'.format(rev, ' -C' if force else '') + return __salt__['cmd.run'](cmd, cwd=cwd) + +def clone(cwd, repository, opts=''): + ''' + Clone a new repository + + Usage:: + + salt '*' hg.clone /path/to/repo https://bitbucket.org/birkenfeld/sphinx + ''' + cmd = 'hg clone {0} {1} {2}'.format(repository, cwd, opts) + return __salt__['cmd.run'](cmd) From cfd92c37338ecdbe2cd343a786cc154fc95228f6 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sun, 11 Mar 2012 00:09:22 -0800 Subject: [PATCH 315/598] Add runas to virtualenv state and module, #670 --- salt/modules/virtualenv.py | 7 +++++-- salt/states/virtualenv.py | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/salt/modules/virtualenv.py b/salt/modules/virtualenv.py index 938fa89ae242..56ec629a642d 100644 --- a/salt/modules/virtualenv.py +++ b/salt/modules/virtualenv.py @@ -17,7 +17,8 @@ def create(path, python='', extra_search_dir='', never_download=False, - prompt=''): + prompt='', + runas=None): ''' Create a virtualenv @@ -40,6 +41,8 @@ def create(path, Passthrough argument given to virtualenv prompt : (default) Passthrough argument given to virtualenv + runas : None + Set ownership for the virtualenv CLI Example:: @@ -62,4 +65,4 @@ def create(path, ' --prompt {0}'.format(prompt) if prompt else '']), path=path) - return __salt__['cmd.run'](cmd) + return __salt__['cmd.run'](cmd, runas=runas) diff --git a/salt/states/virtualenv.py b/salt/states/virtualenv.py index c4566feb9b61..da5edc2d2cad 100644 --- a/salt/states/virtualenv.py +++ b/salt/states/virtualenv.py @@ -17,7 +17,8 @@ def manage(name, extra_search_dir='', never_download=False, prompt='', - __env__='base'): + __env__='base', + runas=None): ''' Create a virtualenv and optionally manage it with pip @@ -75,7 +76,8 @@ def manage(name, python=python, extra_search_dir=extra_search_dir, never_download=never_download, - prompt=prompt) + prompt=prompt, + runas=runas) ret['changes']['new'] = __salt__['cmd.run_stderr']( '{0} -V'.format(venv_py)).strip('\n') From beabd52ecfe2061afc0ccd2c9ba3fb99c2e4b60e Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sun, 11 Mar 2012 16:29:08 -0700 Subject: [PATCH 316/598] Only print tracebacks in salt-call if log-level <= DEBUG --- salt/cli/caller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/cli/caller.py b/salt/cli/caller.py index 22299d5f8a4e..ef9e0b3cb9ea 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -45,7 +45,8 @@ def call(self): ret['return'] = self.minion.functions[fun](*args, **kw) except (TypeError, CommandExecutionError) as exc: msg = 'Error running \'{0}\': {1}\n' - sys.stderr.write(traceback.format_exc()) + if self.opts['log_level'] <= logging.DEBUG: + sys.stderr.write(traceback.format_exc()) sys.stderr.write(msg.format(fun, str(exc))) sys.exit(1) except CommandNotFoundError as exc: From 2ccae78e2512d4913b7af29c9d6f801ba13625d6 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 11 Mar 2012 19:03:25 -0700 Subject: [PATCH 317/598] Fix salt.utils.verify.zmq_version() for zmq 2.1dev --- salt/utils/verify.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 33a898d879fe..dbdadd0d9f07 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -2,10 +2,11 @@ A few checks to make sure the environment is sane ''' # Original Author: Jeff Schroeder <jeffschroeder@computer.org> -import getpass import os -import stat +import re import sys +import stat +import getpass import logging log = logging.getLogger(__name__) @@ -17,11 +18,29 @@ def zmq_version(): '''ZeroMQ python bindings >= 2.1.9 are required''' import zmq ver = zmq.__version__ - ver_int = int(ver.replace('.', '')) - if not ver_int >= 219: - log.critical("ZeroMQ python bindings >= 2.1.9 are required") - return False - return True + # The last matched group can be None if the version + # is something like 3.1 and that will work properly + match = re.match('^(\d+)\.(\d+)(?:\.(\d+))?', ver) + + # Fallthrough and hope for the best + if not match: + msg = 'Using untested zmq python bindings version: \'{0}\'' + log.warn(msg.format(ver)) + return True + + major,minor,point = match.groups() + if major == 2 and minor == 1: + # zmq 2.1dev could be built against a newer libzmq + if "dev" in ver and not point: + log.warn('Using dev zmq module, please report unexpected results') + elif point and point >= 9: + return True + elif major > 2 or (major == 2 and minor > 1): + return True + + # If all else fails, gracefully croak and warn the user + log.critical("ZeroMQ python bindings >= 2.1.9 are required") + return False def verify_env(dirs): From a36e0d63cd017ba8f4a2f05e7da8c5b595306489 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 11 Mar 2012 19:10:33 -0700 Subject: [PATCH 318/598] Test integers, not strings --- salt/utils/verify.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/utils/verify.py b/salt/utils/verify.py index dbdadd0d9f07..def6af684145 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -29,6 +29,16 @@ def zmq_version(): return True major,minor,point = match.groups() + + if major.isdigit(): + major = int(major) + if minor.isdigit(): + minor = int(minor) + + # point very well could be None + if point and point.isdigit(): + point = int(point) + if major == 2 and minor == 1: # zmq 2.1dev could be built against a newer libzmq if "dev" in ver and not point: From e15102eede8fd7fca471d8a5c93197193a044db2 Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Sun, 11 Mar 2012 23:52:39 -0400 Subject: [PATCH 319/598] Raise exception in virt.py if you are unable to contact a hypervisor --- salt/modules/virt.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 42b6420fefd7..a522848acd17 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -32,7 +32,11 @@ def __get_conn(): ''' # This has only been tested on kvm and xen, it needs to be expanded to support # all vm layers supported by libvirt - return libvirt.open("qemu:///system") + conn = libvirt.open("qemu:///system") + if conn == None: + raise Exception('Failed to open a connection to the hypervisor') + else: + return conn def _get_dom(vm_): From 4043ec50594f61229a2fb7460aed4f9c7f3470fb Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Sun, 11 Mar 2012 23:54:51 -0400 Subject: [PATCH 320/598] add an alias (start) for the poorly named create function in virt.py - this matches virsh --- salt/modules/virt.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index a522848acd17..8f1bae52d5cb 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -307,6 +307,17 @@ def create(vm_): return True +def start(vm_): + ''' + Alias for the obscurely named 'create' function + + CLI Example:: + + salt '*' virt.start <vm name> + ''' + return create(vm_) + + def create_xml_str(xml): ''' Start a domain based on the xml passed to the function From bc360c4a2f47ee3af404b98e738e6f3fe22e7138 Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Mon, 12 Mar 2012 00:24:17 -0400 Subject: [PATCH 321/598] fix list_vms to return all vms, running or not. This fixes the create function, among other things --- salt/modules/virt.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 8f1bae52d5cb..22aeaea16f3c 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -78,12 +78,38 @@ def list_vms(): salt '*' virt.list_vms ''' + vms = [] + vms.extend(list_active_vms()) + vms.extend(list_inactive_vms()) + return vms + +def list_active_vms(): + ''' + Return a list of names for active virtual machine on the minion + + CLI Example:: + + salt '*' virt.list_active_vms + ''' conn = __get_conn() vms = [] for id_ in conn.listDomainsID(): vms.append(conn.lookupByID(id_).name()) return vms +def list_inactive_vms(): + ''' + Return a list of names for inactive virtual machine on the minion + + CLI Example:: + + salt '*' virt.list_inactive_vms + ''' + conn = __get_conn() + vms = [] + for id_ in conn.listDefinedDomains(): + vms.append(id_) + return vms def vm_info(): ''' From abcea1b81bb214000625396f948b754cd7519251 Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Mon, 12 Mar 2012 11:45:36 -0400 Subject: [PATCH 322/598] change __get_conn in virt.py to use salt.exceptions --- salt/modules/virt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 22aeaea16f3c..1f98a3a7d04d 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -5,6 +5,7 @@ # of his in the virt func module have been used from xml.dom import minidom +from salt.exceptions import CommandExecutionError import StringIO import os import shutil @@ -34,7 +35,8 @@ def __get_conn(): # all vm layers supported by libvirt conn = libvirt.open("qemu:///system") if conn == None: - raise Exception('Failed to open a connection to the hypervisor') + msg = 'Sorry, {0} failed to open a connection to the hypervisor software' + raise CommandExecutionError(msg.format(__grains__['fqdn'])) else: return conn From 1c8ca22adf01733fae415391934585d2948e689b Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Mon, 12 Mar 2012 12:10:38 -0400 Subject: [PATCH 323/598] actually wrap the connection to libvirt in a try/catch block, before raising the CommandExecutionError exception --- salt/modules/virt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 1f98a3a7d04d..d2645c54cd5d 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -33,12 +33,12 @@ def __get_conn(): ''' # This has only been tested on kvm and xen, it needs to be expanded to support # all vm layers supported by libvirt - conn = libvirt.open("qemu:///system") - if conn == None: + try: + conn = libvirt.open("qemu:///system") + except: msg = 'Sorry, {0} failed to open a connection to the hypervisor software' raise CommandExecutionError(msg.format(__grains__['fqdn'])) - else: - return conn + return conn def _get_dom(vm_): From f4c587218924cbb62cd2e643f394791c789c7f21 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick@nicklang.com> Date: Mon, 12 Mar 2012 19:49:12 -0700 Subject: [PATCH 324/598] changing virtualenv to use no-site-packages by default, since it has been change to be the default behavior --- salt/modules/virtualenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/virtualenv.py b/salt/modules/virtualenv.py index 56ec629a642d..f6678e8ff263 100644 --- a/salt/modules/virtualenv.py +++ b/salt/modules/virtualenv.py @@ -11,7 +11,7 @@ def create(path, venv_bin=__opts__['venv_bin'], - no_site_packages=False, + no_site_packages=True, system_site_packages=False, clear=False, python='', From e45d0155149ccc01d984925234a8dce26fe6cd98 Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Mon, 12 Mar 2012 23:23:19 -0400 Subject: [PATCH 325/598] add localfqdn as core grain, using hostname -f as the source --- salt/grains/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/grains/core.py b/salt/grains/core.py index b6f2c655790a..557d30c83ecc 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -461,6 +461,7 @@ def hostname(): comps = grains['fqdn'].split('.') grains['host'] = comps[0] grains['localhost'] = socket.gethostname() + grains['localfqdn'] = __salt__['cmd.run']('hostname -f').strip() if len(comps) > 1: grains['domain'] = '.'.join(comps[1:]) else: From bd9ae078653147fad05a16a8ce84499c310fa70b Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov <vitty@altlinux.ru> Date: Tue, 13 Mar 2012 11:47:01 +0000 Subject: [PATCH 326/598] Revert "Add all master processes to pidfile" This reverts commit da2df290288f63150c3050f59df36fe83b1d1efd. Conflicts: salt/__init__.py --- salt/__init__.py | 3 ++- salt/master.py | 4 ---- salt/utils/__init__.py | 12 ------------ 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/salt/__init__.py b/salt/__init__.py index fd05304e2211..ec4fac5497d7 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -29,7 +29,8 @@ def __init__(self): # command line overrides config if self.cli['user']: self.opts['user'] = self.cli['user'] - # Send the pidfile location to the opts + + # Send the pidfile location to the opts if self.cli['pidfile']: self.opts['pidfile'] = self.cli['pidfile'] diff --git a/salt/master.py b/salt/master.py index f987ade8c07f..cce27e0702b4 100644 --- a/salt/master.py +++ b/salt/master.py @@ -131,7 +131,6 @@ def _clear_old_jobs(self): ''' Clean out the old jobs ''' - salt.utils.append_pid(self.opts['pidfile']) while True: cur = "{0:%Y%m%d%H}".format(datetime.datetime.now()) @@ -207,7 +206,6 @@ def run(self): ''' Bind to the interface specified in the configuration file ''' - salt.utils.append_pid(self.opts['pidfile']) context = zmq.Context(1) pub_sock = context.socket(zmq.PUB) pull_sock = context.socket(zmq.PULL) @@ -286,7 +284,6 @@ def run(self): ''' Start up the ReqServer ''' - salt.utils.append_pid(self.opts['pidfile']) self.__bind() @@ -376,7 +373,6 @@ def run(self): ''' Start a Master Worker ''' - salt.utils.append_pid(self.opts['pidfile']) self.__bind() diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 0fcf0cda9122..d0e7702c0bc6 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -81,18 +81,6 @@ def get_colors(use=True): return colors -def append_pid(pidfile): - ''' - Save the pidfile - ''' - try: - open(pidfile, 'a').write('\n{0}'.format(str(os.getpid()))) - except IOError: - err = ('Failed to commit the pid to location {0}, please verify' - ' that the location is available').format(pidfile) - log.error(err) - - def daemonize(): ''' Daemonize a process From 8f318549cf86a33bd80f21eb84d1dac445ba0932 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 09:11:41 -0600 Subject: [PATCH 327/598] Fix issue where only one of the group or user is passed --- salt/states/file.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/states/file.py b/salt/states/file.py index 5a92b1188506..7aa940ce84fc 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -338,6 +338,9 @@ def _py(sfn, name, source, user, group, mode, env, context=None): def _check_perms(name, ret, user, group, mode): + ''' + Check the permissions on files and chown if needed + ''' # Check permissions perms = {} perms['luser'] = __salt__['file.get_user'](name) @@ -363,6 +366,10 @@ def _check_perms(name, ret, user, group, mode): perms['cgroup'] = group if 'cuser' in perms or 'cgroup' in perms: if not __opts__['test']: + if user is None: + user = perms['luser'] + if group is None: + group = perms['lgroup'] __salt__['file.chown']( name, user, From 4a5f8d2f3fb4a3cb598066b2c6601dd3a5ab64f2 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 10:19:31 -0600 Subject: [PATCH 328/598] Include docs in source tarball --- MANIFEST.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0276299051b5..e2d4fd6aaa65 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,4 +7,5 @@ recursive-include tests *.py include tests/integration/modules/files/* include tests/integration/files/* include tests/integration/tmp/_README -include tests/unit/templates/files/* \ No newline at end of file +include tests/unit/templates/files/* +recursive-include doc * From a268f6616212bf5ed356cdbfbf4db5fc5e107b22 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 11:03:25 -0600 Subject: [PATCH 329/598] Add check for functions with whitespace --- salt/state.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/state.py b/salt/state.py index 4b0a8a9f98c2..addb474f9a47 100644 --- a/salt/state.py +++ b/salt/state.py @@ -330,6 +330,15 @@ def verify_high(self, high): for arg in body[state]: if isinstance(arg, basestring): fun += 1 + if ' ' in arg.strip(): + errors.append(('The function "{0}" in state ' + '"{1}" in SLS "{2}" has ' + 'whitespace, a function with whitespace is ' + 'not supported, perhaps this is an argument ' + 'that is missing a ":"').format( + arg, + name, + body['__sls__'])) elif isinstance(arg, dict): # The arg is a dict, if the arg is require or # watch, it must be a list. From 21ee588f157de672e425453406165d3dd04b7843 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 12:13:42 -0600 Subject: [PATCH 330/598] Add full data function to publish --- salt/modules/publish.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/salt/modules/publish.py b/salt/modules/publish.py index 95326a6f6712..107f8cec759d 100644 --- a/salt/modules/publish.py +++ b/salt/modules/publish.py @@ -17,8 +17,7 @@ def _get_socket(): socket.connect(__opts__['master_uri']) return socket - -def publish(tgt, fun, arg=None, expr_form='glob', returner='', timeout=5): +def _publish(tgt, fun, arg=None, expr_form='glob', returner='', timeout=5, data='clean'): ''' Publish a command from the minion out to other minions, publications need to be enabled on the Salt master and the minion needs to have permission @@ -62,8 +61,40 @@ def publish(tgt, fun, arg=None, expr_form='glob', returner='', timeout=5): 'ret': returner, 'tok': tok, 'tmo': timeout, + 'form': data, 'id': __opts__['id']} payload['load'] = auth.crypticle.dumps(load) socket = _get_socket() socket.send(serial.dumps(payload)) return auth.crypticle.loads(serial.loads(socket.recv())) + +def publish(tgt, fun, arg=None, expr_form='glob', returner='', timeout=5): + ''' + Publish a command from the minion out to other minions, publications need + to be enabled on the Salt master and the minion needs to have permission + to publish the command. The Salt master will also prevent a recursive + publication loop, this means that a minion cannot command another minion + to command another minion as that would create an infinite command loop. + + The arguments sent to the minion publish function are separated with + commas. This means that for a minion executing a command with multiple + args it will look like this:: + + salt system.example.com publish.publish '*' user.add 'foo,1020,1020' + + CLI Example:: + + salt system.example.com publish.publish '*' cmd.run 'ls -la /tmp' + ''' + return _publish(tgt, fun, arg, expr_form, returner, timeout, 'clean') + +def full_data(tgt, fun, arg=None, expr_form='glob', returner='', timeout=5): + ''' + Return the full data about the publication, this is invoked in the same + way as the publish function + + CLI Example:: + + salt system.example.com publish.full_data '*' cmd.run 'ls -la /tmp' + ''' + return _publish(tgt, fun, arg, expr_form, returner, timeout, 'full') From dcd662ce852bb3f0d93b000f324bc4beb94accf4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 12:15:24 -0600 Subject: [PATCH 331/598] clean up the function def and change kwarg to form --- salt/modules/publish.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/salt/modules/publish.py b/salt/modules/publish.py index 107f8cec759d..de8dd1a238d9 100644 --- a/salt/modules/publish.py +++ b/salt/modules/publish.py @@ -17,7 +17,14 @@ def _get_socket(): socket.connect(__opts__['master_uri']) return socket -def _publish(tgt, fun, arg=None, expr_form='glob', returner='', timeout=5, data='clean'): +def _publish( + tgt, + fun, + arg=None, + expr_form='glob', + returner='', + timeout=5, + form='clean'): ''' Publish a command from the minion out to other minions, publications need to be enabled on the Salt master and the minion needs to have permission @@ -61,7 +68,7 @@ def _publish(tgt, fun, arg=None, expr_form='glob', returner='', timeout=5, data= 'ret': returner, 'tok': tok, 'tmo': timeout, - 'form': data, + 'form': form, 'id': __opts__['id']} payload['load'] = auth.crypticle.dumps(load) socket = _get_socket() From 6bd9a131972ca50e648ea9eef1422d79b18f1b88 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 12:50:03 -0600 Subject: [PATCH 332/598] Add full_returns to publish interactions --- salt/master.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/salt/master.py b/salt/master.py index f987ade8c07f..357b78b2aef6 100644 --- a/salt/master.py +++ b/salt/master.py @@ -711,15 +711,29 @@ def minion_publish(self, clear_load): log.info(('Publishing minion job: #{0[jid]}, func: "{0[fun]}", args:' ' "{0[arg]}", target: "{0[tgt]}"').format(load)) pub_sock.send(self.serial.dumps(payload)) - # Run the client get_returns method - return self.local.get_returns( - jid, - self.local.check_minions( - clear_load['tgt'], - expr_form - ), - timeout - ) + # Run the client get_returns method based on the form data sent + if 'form' in clear_load: + ret_form = clear_load['form'] + else: + ret_form = 'clean' + if ret_form == 'clean': + return self.local.get_returns( + jid, + self.local.check_minions( + clear_load['tgt'], + expr_form + ), + timeout + ) + elif ret_form == 'full': + return self.local.get_full_returns( + jid, + self.local.check_minions( + clear_load['tgt'], + expr_form + ), + timeout + ) def run_func(self, func, load): ''' From 69e2648fbd4eff70eaa215320c1c592bc2c12314 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 12:56:16 -0600 Subject: [PATCH 333/598] Add jid to full data minion publish return --- salt/master.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/master.py b/salt/master.py index 357b78b2aef6..44d689798828 100644 --- a/salt/master.py +++ b/salt/master.py @@ -726,7 +726,7 @@ def minion_publish(self, clear_load): timeout ) elif ret_form == 'full': - return self.local.get_full_returns( + ret = self.local.get_full_returns( jid, self.local.check_minions( clear_load['tgt'], @@ -734,6 +734,8 @@ def minion_publish(self, clear_load): ), timeout ) + ret['__jid__'] = jid + return ret def run_func(self, func, load): ''' From ab0f7236bfdde3ce52d04ce9d19300ff53f2ea70 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 13:05:12 -0600 Subject: [PATCH 334/598] Fix #888 --- salt/minion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index 148c9c5f48c5..3d05c2618e58 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -443,7 +443,6 @@ def tune_in(self): except SaltClientError: # Failed to update the dns, keep the old addr pass - self.passive_refresh() socket.close() socket = context.socket(zmq.SUB) socket.setsockopt(zmq.SUBSCRIBE, '') @@ -451,6 +450,7 @@ def tune_in(self): last = time.time() time.sleep(0.05) multiprocessing.active_children() + self.passive_refresh() else: while True: payload = None From 8ca789ba7d566b3a6024fc6eeb2734c71ce33650 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 13:28:54 -0600 Subject: [PATCH 335/598] Add id to minion publication data --- salt/master.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/master.py b/salt/master.py index 44d689798828..641489481e2d 100644 --- a/salt/master.py +++ b/salt/master.py @@ -689,6 +689,7 @@ def minion_publish(self, clear_load): 'tgt': clear_load['tgt'], 'jid': jid, 'ret': clear_load['ret'], + 'id': clear_load['id'], } expr_form = 'glob' timeout = 5 From 10756d5f169cdd626414efecbcfe5db0926cf8ae Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Tue, 13 Mar 2012 13:53:41 -0700 Subject: [PATCH 336/598] Successfully return from the version check when using a dev copy of zmq. --- salt/utils/verify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/utils/verify.py b/salt/utils/verify.py index def6af684145..9b0c4d8d754e 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -43,6 +43,7 @@ def zmq_version(): # zmq 2.1dev could be built against a newer libzmq if "dev" in ver and not point: log.warn('Using dev zmq module, please report unexpected results') + return True elif point and point >= 9: return True elif major > 2 or (major == 2 and minor > 1): From dc85814aa31349329f0eaa9010b9a147d61c6c38 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 22:01:56 -0600 Subject: [PATCH 337/598] Add test for module kwarg passing --- salt/modules/test.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/salt/modules/test.py b/salt/modules/test.py index a3e25bc97c18..ad6dd45cd720 100644 --- a/salt/modules/test.py +++ b/salt/modules/test.py @@ -72,8 +72,7 @@ def get_opts(): return __opts__ -# FIXME: mutable types as default parameter values -def cross_test(func, args=[]): +def cross_test(func, args=None): ''' Execute a minion function via the __salt__ object in the test module, used to verify that the minion functions can be called @@ -83,9 +82,24 @@ def cross_test(func, args=[]): salt '*' test.cross_test file.gid_to_group 0 ''' + if args is None: + args = [] return __salt__[func](*args) +def kpass(**kwargs): + ''' + Print out the data passed into the function **kwargs, this is used to + both test the publication data and cli kwarg passing, but also to display + the information available within the publication data. + + CLI Example:: + + salt '*' test.kpass + ''' + return kwargs + + def fib(num): ''' Return a Fibonacci sequence up to the passed number, and the From deabc3be5a33f1ec2aaf5b7102ad7b1b21c7e805 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 22:03:02 -0600 Subject: [PATCH 338/598] Pass in publication data via kwargs if the function has it This commit makes it so that module functions that accept **kwargs will have the publication data passed through as __pub_<item>. --- salt/minion.py | 4 ++-- salt/state.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 3d05c2618e58..c543f4f859d5 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -237,7 +237,7 @@ def _thread_return(self, data): if function_name in self.functions: try: func = self.functions[data['fun']] - args, kw = salt.state.build_args(func, data['arg']) + args, kw = salt.state.build_args(func, data['arg'], data) ret['return'] = func(*args, **kw) except CommandNotFoundError as exc: msg = 'Command not found in \'{0}\': {1}' @@ -296,7 +296,7 @@ def _thread_multi_return(self, data): try: func = self.functions[data['fun'][ind]] - args, kw = salt.state.build_args(func, data['arg'][ind]) + args, kw = salt.state.build_args(func, data['arg'][ind], data) ret['return'][data['fun'][ind]] = func(*args, **kw) except Exception as exc: trb = traceback.format_exc() diff --git a/salt/state.py b/salt/state.py index addb474f9a47..30bca0a92ec3 100644 --- a/salt/state.py +++ b/salt/state.py @@ -59,8 +59,11 @@ def _getargs(func): return aspec -def build_args(func, args): - spec_args, _, _, _ = _getargs(func) +def build_args(func, args, data=None): + ''' + Build the args and kwargs + ''' + spec_args, _, kwarg_spec, _ = _getargs(func) _args = [] _kw = {} @@ -76,6 +79,10 @@ def build_args(func, args): _args.append(arg[0]) else: _args.append(arg) + if kwarg_spec and isinstance(data, dict): + # this function accepts kwargs, pack in the publish data + for key, val in data.items(): + _kw['__pub_{0}'.format(key)] = val return _args, _kw From 9136b2eb8552ae4b94b254d1f7af043a2c07d733 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Tue, 13 Mar 2012 21:28:51 -0700 Subject: [PATCH 339/598] Fix a small tyop in the pillar example --- doc/topics/pillar/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/topics/pillar/index.rst b/doc/topics/pillar/index.rst index 900fe3c4c652..baa1c451b3af 100644 --- a/doc/topics/pillar/index.rst +++ b/doc/topics/pillar/index.rst @@ -45,7 +45,6 @@ found in the packages sls file: {% if grains['os'] == 'RedHat' %} apache: httpd git: git - {% endif %} {% elif grains['os'] == 'Debian' %} apache: apache2 git: git-core From a5159c840eff62172c15b030a9c64c9ac42b7c4d Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Wed, 14 Mar 2012 00:34:37 -0400 Subject: [PATCH 340/598] another typo --- doc/topics/pillar/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/pillar/index.rst b/doc/topics/pillar/index.rst index baa1c451b3af..c956ce516939 100644 --- a/doc/topics/pillar/index.rst +++ b/doc/topics/pillar/index.rst @@ -37,7 +37,7 @@ used for states, and has the same structure: '*': - packages -This sim[ple pillar top file declares that information for all minions can be +This simple pillar top file declares that information for all minions can be found in the packages sls file: .. code-block:: yaml From 51dc7c26dff66e9264b9dc6a183f4959d4f89bdc Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Wed, 14 Mar 2012 00:39:47 -0400 Subject: [PATCH 341/598] apostrophes are hard --- doc/topics/pillar/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/pillar/index.rst b/doc/topics/pillar/index.rst index c956ce516939..256e52d61b98 100644 --- a/doc/topics/pillar/index.rst +++ b/doc/topics/pillar/index.rst @@ -38,7 +38,7 @@ used for states, and has the same structure: - packages This simple pillar top file declares that information for all minions can be -found in the packages sls file: +found in the package's sls file: .. code-block:: yaml From fc3acae5868525c7dd165245fa162c302c02d7dc Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 22:56:07 -0600 Subject: [PATCH 342/598] Add check for refreshing the pillar data int he module_refresh file --- salt/minion.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/minion.py b/salt/minion.py index c543f4f859d5..2c9069167869 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -411,6 +411,15 @@ def passive_refresh(self): ''' fn_ = os.path.join(self.opts['cachedir'], 'module_refresh') if os.path.isfile(fn_): + with open(fn_, 'r+') as f: + data = f.read() + if 'pillar' in data: + self.opts['pillar'] = salt.pillar.get_pillar( + self.opts, + self.opts['grains'], + self.opts['id'], + self.opts['environment'], + ).compile_pillar() os.remove(fn_) self.functions, self.returners = self.__load_modules() From 7de557d64cc1f6ff3be71f88615b48111d75e990 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 23:01:16 -0600 Subject: [PATCH 343/598] change test.kpass to test.kwarg --- salt/modules/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/test.py b/salt/modules/test.py index ad6dd45cd720..c6d16745cef1 100644 --- a/salt/modules/test.py +++ b/salt/modules/test.py @@ -87,7 +87,7 @@ def cross_test(func, args=None): return __salt__[func](*args) -def kpass(**kwargs): +def kwarg(**kwargs): ''' Print out the data passed into the function **kwargs, this is used to both test the publication data and cli kwarg passing, but also to display @@ -95,7 +95,7 @@ def kpass(**kwargs): CLI Example:: - salt '*' test.kpass + salt '*' test.kwarg ''' return kwargs From 0e66b77fcf2ef6b8980898dcf42c6e8a374c9a89 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Tue, 13 Mar 2012 22:12:23 -0700 Subject: [PATCH 344/598] A few small changes to the cmd module before renaming the file --- salt/modules/cmd.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/modules/cmd.py b/salt/modules/cmd.py index fe668e100528..72262dd6a975 100644 --- a/salt/modules/cmd.py +++ b/salt/modules/cmd.py @@ -11,6 +11,8 @@ import tempfile import salt.utils from salt.exceptions import CommandExecutionError + +# Only available on posix systems, nonfatal on windows try: import pwd except: @@ -28,6 +30,13 @@ DEFAULT_SHELL = '/bin/sh' +def __virtual__(): + ''' + Overwriting the cmd python module makes debugging modules + with pdb a bit harder so lets do it this way instead. + ''' + return 'cmd' + def _run(cmd, cwd=None, From 18010c2cf632c5d1c4c54947259d8d7fca398e90 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Tue, 13 Mar 2012 22:14:39 -0700 Subject: [PATCH 345/598] Rename the cmd module file to cmdmod to not overwrite stdlib This makes debugging other modules with pdb when developing hard --- salt/modules/{cmd.py => cmdmod.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename salt/modules/{cmd.py => cmdmod.py} (100%) diff --git a/salt/modules/cmd.py b/salt/modules/cmdmod.py similarity index 100% rename from salt/modules/cmd.py rename to salt/modules/cmdmod.py From 7079dbf04ffe6e02bf3b89a41507494969a2c990 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 13 Mar 2012 23:19:48 -0600 Subject: [PATCH 346/598] Add saltutil.refresh_pillar --- salt/modules/saltutil.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index 3bc2ec25ba97..a29c197ba007 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -47,7 +47,9 @@ def _sync(form, env): shutil.copyfile(fn_, dest) ret.append('{0}.{1}'.format(form, os.path.basename(fn_))) if ret: - open(os.path.join(__opts__['cachedir'], 'module_refresh'), 'w+').write('') + mod_file = os.path.join(__opts__['cachedir'], 'module_refresh') + with open(mod_file, 'a+') as f: + f.write('') if __opts__.get('clean_dynamic_modules', True): current = set(os.listdir(mod_dir)) for fn_ in current - remote: @@ -145,6 +147,23 @@ def sync_all(env='base'): return ret +def refresh_pillar(): + ''' + Queue the minion to refresh the pillar data. + + CLI Example:: + + salt '*' saltutil.refresh_pillar + ''' + mod_file = os.path.join(__opts__['cachedir'], 'module_refresh') + try: + with open(mod_file, 'a+') as f: + f.write('pillar') + return True + except IOError: + return False + + def running(): ''' Return the data on all running processes salt on the minion From 081d251086428836f1d7dbe907287008c6307b40 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Tue, 13 Mar 2012 22:41:51 -0700 Subject: [PATCH 347/598] Start documenting optional dependencies and make win_file use it The new "standard" for documenting non-stdlib python dependencies is via a module level docstring that includes a line like: Required python modules: module1, someothermodule, win32api Add a few helper methods for returning a list of required python modules which aren't importable and pretty print the error in win_file --- salt/modules/win_file.py | 33 +++++++++++++++++++++---------- salt/utils/__init__.py | 42 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py index dd93171023db..828febdc4594 100644 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -1,26 +1,39 @@ ''' Manage information about files on the minion, set/read user, group data + +Required python modules: win32api, win32con, win32security, ntsecuritycon ''' import os -import os.path import time +import os.path import hashlib -import win32api -import win32con -import win32security -import ntsecuritycon as con - -#import salt.utils.find +import logging + +try: + import win32api + import win32con + import win32security + import ntsecuritycon as con + has_windows_modules = True +except ImportError: + has_windows_modules = False + +import salt.utils from salt.exceptions import SaltInvocationError +log = logging.getLogger(__name__) + def __virtual__(): ''' Only works on Windows systems ''' if __grains__['os'] == 'Windows': - return 'file' + if has_windows_modules: + return 'file' + log.warn(salt.utils.required_modules_error(__file__, __doc__)) + return False return False __outputter__ = { @@ -63,7 +76,7 @@ def get_gid(path): salt '*' file.get_gid c:\\temp\\test.txt ''' if not os.path.exists(path): - return False + return False secdesc = win32security.GetFileSecurity (path, win32security.OWNER_SECURITY_INFORMATION) owner_sid = secdesc.GetSecurityDescriptorOwner() return win32security.ConvertSidToStringSid(owner_sid) @@ -104,7 +117,7 @@ def user_to_uid(user): CLI Example:: - salt '*' file.user_to_uid myusername + salt '*' file.user_to_uid myusername ''' sid, domain, type = win32security.LookupAccountName (None, user) return win32security.ConvertSidToStringSid(sid) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 0fcf0cda9122..790df1336fe4 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -2,11 +2,11 @@ Some of the utils used by salt ''' -from calendar import month_abbr as months -import logging import os -import socket import sys +import socket +import logging +from calendar import month_abbr as months from salt.exceptions import SaltClientError @@ -277,3 +277,39 @@ def dns_check(addr, safe=False): sys.stderr.write(err) sys.exit(42) return addr + + +def required_module_list(docstring=None): + ''' + Return a list of python modules required by a salt module that aren't + in stdlib and don't exist on the current pythonpath. + + NOTE: this function expects docstring to include something like: + Required python modules: win32api, win32con, win32security, ntsecuritycon + ''' + ret = [] + txt = 'Required python modules: ' + data = docstring.split('\n') if docstring else [] + mod_list = filter(lambda x: x.startswith(txt), data) + if not mod_list: + return [] + modules = mod_list[0].replace(txt, '').split(', ') + for mod in modules: + try: + __import__(mod) + except ImportError: + ret.append(mod) + return ret + + +def required_modules_error(name, docstring): + ''' + Pretty print error messages in critical salt modules which are + missing deps not always in stdlib such as win32api on windows. + ''' + modules = required_module_list(docstring) + if not modules: + return '' + filename = os.path.basename(name).split('.')[0] + msg = '\'{0}\' requires these python modules: {1}' + return msg.format(filename, ', '.join(modules)) From f701bb8302c10d0c5a41b1658617784e39df4926 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Tue, 13 Mar 2012 22:56:08 -0700 Subject: [PATCH 348/598] Add "Required python modules" to the windows registry module Also just use a fallthrough in the win_file module --- salt/modules/reg.py | 16 +++++++++++++--- salt/modules/win_file.py | 1 - 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/salt/modules/reg.py b/salt/modules/reg.py index ba3e5b3a18f5..c457fccfc0df 100644 --- a/salt/modules/reg.py +++ b/salt/modules/reg.py @@ -1,15 +1,25 @@ ''' Manage the registry on Windows + +Required python modules: _winreg ''' -import _winreg +try: + import _winreg + has_windows_modules = True +except ImportError: + has_windows_modules = False + +import salt.utils def __virtual__(): ''' Only works on Windows systems ''' if __grains__['os'] == 'Windows': - return 'reg' + if has_windows_modules: + return 'reg' + log.warn(salt.utils.required_modules_error(__file__, __doc__)) return False @@ -88,7 +98,7 @@ def delete_key(hkey, path, key): CLI Example:: - salt '*' reg.delete_key HKEY_CURRENT_USER 'SOFTWARE\\Salt' 'version' + salt '*' reg.delete_key HKEY_CURRENT_USER 'SOFTWARE\\Salt' 'version' ''' hkey2 = hkeys[hkey] try: diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py index 828febdc4594..e37abe87acd2 100644 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -33,7 +33,6 @@ def __virtual__(): if has_windows_modules: return 'file' log.warn(salt.utils.required_modules_error(__file__, __doc__)) - return False return False __outputter__ = { From d9e744e1c389a79daf4ac7962e757dd9301eaa94 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov <vitty@altlinux.ru> Date: Wed, 14 Mar 2012 11:41:11 +0000 Subject: [PATCH 349/598] If we switch to another user pidfiles must be written from original one (possibly root) as pidfiles in /var/run are usually owned by this original user. --- salt/__init__.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/salt/__init__.py b/salt/__init__.py index ec4fac5497d7..61b98e3f9bfc 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -98,14 +98,14 @@ def start(self): import logging log = logging.getLogger(__name__) # Late import so logging works correctly + import salt.master + master = salt.master.Master(self.opts) + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + set_pidfile(self.opts['pidfile']) if check_user(self.opts['user'], log): - import salt.master - master = salt.master.Master(self.opts) - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - set_pidfile(self.opts['pidfile']) try: master.start() except salt.master.MasterExit: @@ -187,13 +187,13 @@ def start(self): # Late import so logging works correctly import salt.minion log = logging.getLogger(__name__) + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + set_pidfile(self.cli['pidfile']) if check_user(self.opts['user'], log): try: - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - set_pidfile(self.cli['pidfile']) minion = salt.minion.Minion(self.opts) minion.tune_in() except KeyboardInterrupt: @@ -304,14 +304,14 @@ def start(self): # Late import so logging works correctly import salt.minion log = logging.getLogger(__name__) + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + set_pidfile(self.cli['pidfile']) if check_user(self.opts['user'], log): try: syndic = salt.minion.Syndic(self.opts) - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - set_pidfile(self.cli['pidfile']) syndic.tune_in() except KeyboardInterrupt: log.warn('Stopping the Salt Syndic Minion') From eaa0c5ec921619418273c37118e09c5d9d4b6de2 Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Wed, 14 Mar 2012 11:40:37 -0600 Subject: [PATCH 350/598] Fix small typo --- doc/ref/file_server/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ref/file_server/index.rst b/doc/ref/file_server/index.rst index 261036bcc362..5c5358eda38c 100644 --- a/doc/ref/file_server/index.rst +++ b/doc/ref/file_server/index.rst @@ -6,7 +6,7 @@ Salt comes with a simple file server suitable for distributing files to the salt minions. The file server is a stateless ZeroMQ server that is built into the salt master. -The main intent of the Salt File server is the present files for use in the +The main intent of the Salt File server is to present files for use in the Salt state system. With this said, the Salt file server can be used for any general file transfer from the master to the minions. From 299843245d2adfaeb5910a8a99ba352957914ad4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 14 Mar 2012 12:35:45 -0600 Subject: [PATCH 351/598] fix grains/core.py static reference to cmd.run --- salt/grains/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 557d30c83ecc..9f25161bfabf 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -22,8 +22,8 @@ # Solve the Chicken and egg problem where grains need to run before any # of the modules are loaded and are generally available for any usage. -import salt.modules.cmd -__salt__ = {'cmd.run': salt.modules.cmd._run_quiet} +import salt.modules.cmdmod +__salt__ = {'cmd.run': salt.modules.cmdmod._run_quiet} def _kernel(): From 6c10c17fbfe7f1768424ef7e832bafa64bc876e7 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 14 Mar 2012 14:24:41 -0600 Subject: [PATCH 352/598] pull out executable on windows --- salt/modules/cmdmod.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 72262dd6a975..7defc5f2e9a7 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -94,15 +94,15 @@ def _run(cmd, run_env = os.environ run_env.update(env) + kwargs = {'cwd': cwd, + 'shell': True, + 'env': run_env, + 'stdout': stdout, + 'stderr':stderr} + if not os.environ.get('os', '').startswith('Windows'): + kwargs['executable'] = shell # This is where the magic happens - proc = subprocess.Popen(cmd, - executable=shell, - cwd=cwd, - shell=True, - env=run_env, - stdout=stdout, - stderr=stderr - ) + proc = subprocess.Popen(cmd, **kwargs) out = proc.communicate() ret['stdout'] = out[0] From 4481b26c9b209ad90fc1b4a94eac3a2b2768325e Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 14 Mar 2012 15:07:17 -0600 Subject: [PATCH 353/598] Add providers to the capabilities of the minion config --- salt/loader.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/salt/loader.py b/salt/loader.py index f20c69739419..8f18de672758 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -44,7 +44,21 @@ def minion_mods(opts): Returns the minion modules ''' load = _create_loader(opts, 'modules', 'module') - return load.apply_introspection(load.gen_functions()) + functions = load.apply_introspection(load.gen_functions()) + if opts.get('providers', False): + if isinstance(opts['providers'], dict): + for mod, provider in opts['providers'].items(): + funcs = raw_mod(opts, + provider, + functions) + if funcs: + for func in funcs: + f_key = '{0}{1}'.format( + mod, + func[func.rindex('.'):] + ) + functions[f_key] = funcs[func] + return functions def raw_mod(opts, name, functions): From 2003cd7b42755713b2d8eff4079b6417f3693552 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 14 Mar 2012 15:11:11 -0600 Subject: [PATCH 354/598] Add providers blurb to minion config --- conf/minion.template | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/conf/minion.template b/conf/minion.template index 7fac17c528f8..46b89c5bf3d0 100644 --- a/conf/minion.template +++ b/conf/minion.template @@ -66,6 +66,14 @@ #states_dirs: [] #render_dirs: [] # +# A module provider can be statically overwritten or extended for the minion +# via the providers option, in this case the default module will be +# overwritten by the specified module. In this example the pkg module will +# be provided by the yumpkg5 module instead of the system default. +# +# providers: +# pkg: yumpkg5 +# # Enable Cython modules searching and loading. (Default: False) #cython_enable: False From 2ff79fee22481a8703fe240f0346e6b48bb2527f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 14 Mar 2012 15:12:02 -0600 Subject: [PATCH 355/598] Add providers to config.py --- salt/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/config.py b/salt/config.py index 36e43ab956d8..08ad99e0dc78 100644 --- a/salt/config.py +++ b/salt/config.py @@ -143,6 +143,7 @@ def minion_config(path): 'returner_dirs': [], 'states_dirs': [], 'render_dirs': [], + 'providers': {}, 'clean_dynamic_modules': True, 'open_mode': False, 'multiprocessing': True, From e297628db24a80c16b0bf0867206d913b79e1b06 Mon Sep 17 00:00:00 2001 From: Gordon McAllister <gordon.mcallister@gmail.com> Date: Wed, 14 Mar 2012 17:07:46 -0700 Subject: [PATCH 356/598] Fix show_lowstate example in docstring --- salt/modules/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index 97dbbe4e8cc4..79d331818fc6 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -124,7 +124,7 @@ def show_lowstate(): CLI Example:: - salt '*' show_lowstate + salt '*' state.show_lowstate ''' st_ = salt.state.HighState(__opts__) return st_.compile_low_chunks() From 72c2014767bb6f117afa8439d83f6e271bce7d32 Mon Sep 17 00:00:00 2001 From: Gordon McAllister <gordon.mcallister@gmail.com> Date: Wed, 14 Mar 2012 17:09:19 -0700 Subject: [PATCH 357/598] War on typos Also cleanup whitespace and break some long lines --- doc/ref/cli/index.rst | 4 ++-- doc/ref/cli/salt.rst | 2 +- doc/ref/configuration/master.rst | 2 +- doc/ref/configuration/minion.rst | 6 ++--- doc/ref/file_server/dynamic-modules.rst | 2 +- doc/ref/file_server/file_roots.rst | 2 +- doc/ref/file_server/index.rst | 2 +- doc/ref/modules/index.rst | 4 ++-- doc/ref/python-api.rst | 2 +- doc/ref/returners/index.rst | 2 +- doc/ref/runners.rst | 2 +- doc/ref/states/highstate.rst | 8 +++---- doc/ref/states/index.rst | 4 ++-- doc/ref/states/ordering.rst | 4 ++-- doc/ref/states/top.rst | 2 +- doc/ref/states/writing.rst | 6 ++--- doc/ref/syndic.rst | 8 +++---- doc/topics/installation/arch.rst | 13 +++++----- doc/topics/installation/fedora.rst | 6 ++--- doc/topics/installation/freebsd.rst | 9 +++---- doc/topics/pillar/index.rst | 4 ++-- doc/topics/specs/salt_auth_proto.rst | 2 +- doc/topics/specs/salt_auth_proto_abs.rst | 22 ++++++++--------- doc/topics/targeting/grains.rst | 20 ++++++++-------- doc/topics/troubleshooting/index.rst | 14 +++++------ .../troubleshooting/yaml_idiosyncrasies.rst | 11 ++++----- doc/topics/tutorials/bootstrap_ec2.rst | 24 +++++++++++++++---- 27 files changed, 101 insertions(+), 86 deletions(-) diff --git a/doc/ref/cli/index.rst b/doc/ref/cli/index.rst index 90b03f1579ae..39e281003c19 100644 --- a/doc/ref/cli/index.rst +++ b/doc/ref/cli/index.rst @@ -2,9 +2,9 @@ Command Line Reference ====================== -Salt can be controlled by a command line client by the root user on the Salt +Salt can be controlled by a command line client by the root user on the Salt master. The Salt command line client uses the Salt client API to communicate -with the Salt master server. The Salt client is straightforward and simple +with the Salt master server. The Salt client is straightforward and simple to use. Using the Salt client commands can be easily sent to the minions. diff --git a/doc/ref/cli/salt.rst b/doc/ref/cli/salt.rst index 0f872fe51218..15c7f7fb0415 100644 --- a/doc/ref/cli/salt.rst +++ b/doc/ref/cli/salt.rst @@ -99,7 +99,7 @@ Options form, this is suitable for re-reading the output into an executing python script with eval. -.. option:: --text-out +.. option:: --text-out Print the output from the salt command in the same form the shell would. diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 1876e3a1c607..cf77e5b2d600 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -206,7 +206,7 @@ root of the base environment Default: None The external_nodes option allows Salt to gather data that would normally be -placed in a top file from and external node controller. The external_nodes +placed in a top file from and external node controller. The external_nodes option is the executable that will return the ENC data. Remember that Salt will look for external nodes AND top files and combine the results if both are enabled and available! diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 0146c84c5b92..5cc39eef6cff 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -91,7 +91,7 @@ clusters. ``sub_timeout`` --------------- -The minion connection to the master may be inturupted, the minion will +The minion connection to the master may be interrupted, the minion will verify the connection every so many seconds, to disable connection verification set this value to 0 @@ -172,7 +172,7 @@ disabling modules will lover the minion's ram footprint. Default: ``[]`` (all returners are enabled by default) -If certian returners should be disabled, this is the place +If certain returners should be disabled, this is the place .. code-block:: yaml @@ -274,7 +274,7 @@ The default renderer used for local state executions Default: ``False`` state_verbose allows for the data returned from the minion to be more -verbose. Normaly only states that fail or states that have changes are +verbose. Normally only states that fail or states that have changes are returned, but setting state_verbose to True will return all states that were checked diff --git a/doc/ref/file_server/dynamic-modules.rst b/doc/ref/file_server/dynamic-modules.rst index c204d59d8790..81cf7f315152 100644 --- a/doc/ref/file_server/dynamic-modules.rst +++ b/doc/ref/file_server/dynamic-modules.rst @@ -18,7 +18,7 @@ The directories are prepended with an underscore: The contents of these directories need to be synced over to the minions after python modules have been created in them. There are a number of ways to sync -the modules. +the modules. Sync Via States =============== diff --git a/doc/ref/file_server/file_roots.rst b/doc/ref/file_server/file_roots.rst index 96edaa6b64ef..1b13d3126830 100644 --- a/doc/ref/file_server/file_roots.rst +++ b/doc/ref/file_server/file_roots.rst @@ -6,7 +6,7 @@ The Salt file server is a high performance file server written in ZeroMQ. It manages large files quickly and with little overhead, and has been optimized to handle small files in an extremely efficient manner. -The Salt file server is an environment aware file server, this means that +The Salt file server is an environment aware file server, this means that files can be allocated within many root directories and accessed by specifying both the file path and the environment to search. The individual environments can also be spanned across multiple directory roots diff --git a/doc/ref/file_server/index.rst b/doc/ref/file_server/index.rst index 5c5358eda38c..03e8fde7a0ce 100644 --- a/doc/ref/file_server/index.rst +++ b/doc/ref/file_server/index.rst @@ -4,7 +4,7 @@ Salt File Server Salt comes with a simple file server suitable for distributing files to the salt minions. The file server is a stateless ZeroMQ server that is built into -the salt master. +the salt master. The main intent of the Salt File server is to present files for use in the Salt state system. With this said, the Salt file server can be used for any diff --git a/doc/ref/modules/index.rst b/doc/ref/modules/index.rst index 2680e7ac9f68..656cd2478b2b 100644 --- a/doc/ref/modules/index.rst +++ b/doc/ref/modules/index.rst @@ -32,7 +32,7 @@ This means that when creating a module functions in modules which already exist can be called. The variable ``__salt__`` is packed into the modules after they are loaded into -the salt minion. This variable is a python dictionary of all of the salt +the salt minion. This variable is a python dictionary of all of the salt functions, laid out in the same way that they are made available to the salt command. @@ -40,7 +40,7 @@ Salt modules can be cross called by accessing the value in the ``__salt__`` dict: .. code-block:: python - + def foo(bar): return __salt__['cmd.run'](bar) diff --git a/doc/ref/python-api.rst b/doc/ref/python-api.rst index 86fe430bc6b9..9f37ba93e182 100644 --- a/doc/ref/python-api.rst +++ b/doc/ref/python-api.rst @@ -65,7 +65,7 @@ Compound Command Execution With the Salt API The Salt client API can also send what is called a compound command. Often a collection of commands need to be executed on the targeted minions, rather than send the commands one after another, they can be send in a single publish -containing a series of commands. This can dramatically lower overhead and +containing a series of commands. This can dramatically lower overhead and speed up the application communicating with Salt. When commands are executed with compound execution the minion functions called diff --git a/doc/ref/returners/index.rst b/doc/ref/returners/index.rst index 0a77a4fe1b3a..d6b987fd2a89 100644 --- a/doc/ref/returners/index.rst +++ b/doc/ref/returners/index.rst @@ -38,7 +38,7 @@ It is also possible to specify multiple returners: salt '*' test.ping --return mongo_return,redis_return,cassandra_return In this scenario all three returners will be called and the data from the -test.ping command will be sent out to the three named returers. +test.ping command will be sent out to the three named returners. Writing a Returner ================== diff --git a/doc/ref/runners.rst b/doc/ref/runners.rst index 138866813db1..bbd4d2b11e51 100644 --- a/doc/ref/runners.rst +++ b/doc/ref/runners.rst @@ -32,7 +32,7 @@ The best examples of runners can be found in the Salt source: :blob:`salt/runners` -A simple runner that returns a well formated list of the minons that are +A simple runner that returns a well-formatted list of the minions that are responding to salt calls would look like this: .. code-block:: python diff --git a/doc/ref/states/highstate.rst b/doc/ref/states/highstate.rst index 188960642a94..718e24570ac8 100644 --- a/doc/ref/states/highstate.rst +++ b/doc/ref/states/highstate.rst @@ -14,7 +14,7 @@ The Salt State Tree Configurable via :conf_master:`state_top`. .. seealso:: :doc:`A detailed description of the top file </ref/states/top>` - + .. glossary:: State tree @@ -98,7 +98,7 @@ declaration that will restart Apache whenever the Apache configuration file, include: - apache - + extend: apache: service: @@ -108,7 +108,7 @@ declaration that will restart Apache whenever the Apache configuration file, mywebsite: file: - managed - + State declaration ----------------- @@ -277,7 +277,7 @@ For example, given the following state declaration: - python-yaml Once converted into the :term:`lowstate` data structure the above state -declaration will be expaneded into the following three state declarations: +declaration will be expanded into the following three state declarations: .. code-block:: yaml diff --git a/doc/ref/states/index.rst b/doc/ref/states/index.rst index cd838e2aaff5..707499af7f3e 100644 --- a/doc/ref/states/index.rst +++ b/doc/ref/states/index.rst @@ -121,7 +121,7 @@ Here is an example of a Salt State: - salt-minion - watch: - file: /etc/salt/minion - + /etc/salt/minion: file: - managed @@ -145,7 +145,7 @@ The top file is the mapping for the state system. The top file specifies which minions should have which modules applied and which environments they should draw the states from. -The top file works by specifying the environment, containing matchers with +The top file works by specifying the environment, containing matchers with lists of Salt states sent to the matching minions: .. code-block:: yaml diff --git a/doc/ref/states/ordering.rst b/doc/ref/states/ordering.rst index a1ab1a088c8d..18c4d2a82449 100644 --- a/doc/ref/states/ordering.rst +++ b/doc/ref/states/ordering.rst @@ -9,7 +9,7 @@ order is not necessarily defined the way you want it. A few tools exist in Salt to set up the correct state ordering. These tools consist of requisite declarations and order options. -.. note:: +.. note:: Salt does **not** execute :term:`state declarations <state declaration>` in the order they appear in the source. @@ -149,7 +149,7 @@ restarted. Watch and the Watcher Function ------------------------------ -The watch requisite is based on the ``watcher`` function, state python +The watch requisite is based on the ``watcher`` function, state python modules can include a function called watcher, this function is then called if the watch call is invoked. In the case of the service module the underlying service is restarted. In the case of the cmd state the command is executed. diff --git a/doc/ref/states/top.rst b/doc/ref/states/top.rst index 38e9297efb6b..e2ab4420bfcb 100644 --- a/doc/ref/states/top.rst +++ b/doc/ref/states/top.rst @@ -121,7 +121,7 @@ A clean and recommended setup for multiple environments would look like this: Then only place state trees in the dev, qa and prod environments, leaving the base environment open for generic file transfers. Then the top.sls file would look something like this: - + .. code-block:: yaml dev: diff --git a/doc/ref/states/writing.rst b/doc/ref/states/writing.rst index 955922269a05..7677706c3470 100644 --- a/doc/ref/states/writing.rst +++ b/doc/ref/states/writing.rst @@ -34,8 +34,8 @@ directly define the user interface. Using Custom State Modules ========================== -Place your custom state modules inside a ``_states`` directory within the -``file_roots`` specified by the master config file. These custome state modules +Place your custom state modules inside a ``_states`` directory within the +``file_roots`` specified by the master config file. These custom state modules can then be distributed in a number of ways. Custom state modules are distributed when state.highstate is run, or via the saltutil.sync_states function. @@ -114,7 +114,7 @@ executing state as an argument. The low state data is a dict and can be seen by executing the state.show_lowstate function. Then the mod_init function must return a bool. If the return value is True, then the mod_init function will not be executed again, meaning that the needed behavior has been set up. Otherwise, -if the mod_init function returns False, then the function will be called the +if the mod_init function returns False, then the function will be called the next time. A good example of the mod_init function is found in the pkg state module: diff --git a/doc/ref/syndic.rst b/doc/ref/syndic.rst index a7c40d72f577..f8fd61955b92 100644 --- a/doc/ref/syndic.rst +++ b/doc/ref/syndic.rst @@ -6,7 +6,7 @@ The Salt Syndic interface is a powerful tool which allows for the construction of Salt command topologies. A basic Salt setup has a Salt Master commanding a group of Salt Minions. The Syndic interface is a special passthrough minion, it is run on a master and connects to another master, then the master -that the Syndic minion is listening to can control the minions attatched to +that the Syndic minion is listening to can control the minions attached to the master running the syndic. The intent for supporting many layouts is not presented with the intent of @@ -16,7 +16,7 @@ of controlling many systems. Configuring the Syndic ====================== -Since the Syndic only needs to be attatched to a higher level master the +Since the Syndic only needs to be attached to a higher level master the configuration is very simple. On a master that is running a syndic to connect to a higher level master the syndic_master option needs to be set in the master config file. The syndic_master option contains the hostname or ip @@ -25,7 +25,7 @@ running on. The master that the syndic connects to sees the syndic as an ordinary minion, and treats it as such. the higher level master will need to accept the syndic's -minion key like any other minion. This master will also need to set the +minion key like any other minion. This master will also need to set the order_masters value in the configuration to True. The order_masters option in the config on the higher level master is very important, to control a syndic extra information needs to be sent with the publications, the order_masters @@ -34,7 +34,7 @@ option makes sure that the extra data is sent out. Running the Syndic ================== -The Syndic is a seperate daemon that needs to be started on the master that is +The Syndic is a separate daemon that needs to be started on the master that is controlled by a higher master. Starting the Syndic daemon is the same as starting the other Salt daemons. diff --git a/doc/topics/installation/arch.rst b/doc/topics/installation/arch.rst index 3d6d30f979ce..0f0af1837619 100644 --- a/doc/topics/installation/arch.rst +++ b/doc/topics/installation/arch.rst @@ -2,10 +2,10 @@ Arch Linux ========== -Salt has primarily been developed on Arch Linux, meaning it is known to work -very well on that distribution. The lead developer, Thomas S. Hatch (thatch45) has -been a TU (Trusted User) for the Arch Linux distribution, and has written a -number of Arch-specific tools in the past. +Salt has primarily been developed on Arch Linux, meaning it is known to +work very well on that distribution. The lead developer, Thomas S. Hatch +(thatch45) has been a TU (Trusted User) for the Arch Linux distribution, +and has written a number of Arch-specific tools in the past. Salt, while not Arch-specific, is packaged for and works well on Arch Linux. @@ -50,8 +50,9 @@ currently relies on the following packages only available via the AUR: Tracking develop ---------------- -To install the bleeding edge version of Salt (**may include bugs!**), you can use -the -git package. Installing the -git package can be done using the commands: +To install the bleeding edge version of Salt (**may include bugs!**), +you can use the -git package. Installing the -git package can be done +using the commands: .. code-block:: bash diff --git a/doc/topics/installation/fedora.rst b/doc/topics/installation/fedora.rst index af74afab1562..338c9b2cc9e5 100644 --- a/doc/topics/installation/fedora.rst +++ b/doc/topics/installation/fedora.rst @@ -150,9 +150,9 @@ Minion. This ensures that the commands you send to your Minions (your cloud) can not be tampered with, and that communication between Master and Minion is only done through trusted, accepted keys. -Before you'll be able to do any remote execution or configuration management you'll -need to accept any pending keys on the Master. Run the ``salt-key`` command to -list the keys known to the Salt Master: +Before you'll be able to do any remote execution or configuration management +you'll need to accept any pending keys on the Master. Run the ``salt-key`` +command to list the keys known to the Salt Master: .. code-block:: bash diff --git a/doc/topics/installation/freebsd.rst b/doc/topics/installation/freebsd.rst index 69f23c42a380..e0e3cf612a41 100644 --- a/doc/topics/installation/freebsd.rst +++ b/doc/topics/installation/freebsd.rst @@ -35,10 +35,11 @@ Configuration In the sections below I'll outline configuration options for both the Salt Master and Salt Minions. -The Salt port installs two sample configuration files, ``salt/master.sample`` and -``salt/minion.sample`` (these should be installed in ``/usr/local/etc/``, unless you use a -different ``%%PREFIX%%``). You'll need to copy these .sample files into place and -make a few edits. First, copy them into place as seen here: +The Salt port installs two sample configuration files, ``salt/master.sample`` +and ``salt/minion.sample`` (these should be installed in ``/usr/local/etc/``, +unless you use a different ``%%PREFIX%%``). You'll need to copy these +.sample files into place and make a few edits. First, copy them into place +as seen here: .. code-block:: bash diff --git a/doc/topics/pillar/index.rst b/doc/topics/pillar/index.rst index 256e52d61b98..c94eae8003b0 100644 --- a/doc/topics/pillar/index.rst +++ b/doc/topics/pillar/index.rst @@ -12,13 +12,13 @@ Declaring the Master Pillar =========================== The Salt Master server maintains a pillar_roots setup that matches the -structure of the file_roots used in the Salt file server. Like the +structure of the file_roots used in the Salt file server. Like the Salt file server the ``pillar_roots`` option in the master config is based on environments mapping to directories. The pillar data is then mapped to minions based on matchers in a top file which is laid out in the same way as the state top file. -the configuration for the pillar_roots in the master config is identical in +The configuration for the pillar_roots in the master config is identical in behavior and function as the file_roots configuration: .. code-block:: yaml diff --git a/doc/topics/specs/salt_auth_proto.rst b/doc/topics/specs/salt_auth_proto.rst index c183acb06201..85ad5456b185 100644 --- a/doc/topics/specs/salt_auth_proto.rst +++ b/doc/topics/specs/salt_auth_proto.rst @@ -4,7 +4,7 @@ Salt Authentication Protocol The Salt Authentication Protocol (SAP) defines a viable mechanism to create an authenticated, encrypted communication channel. The SAP is used to create a -general purpose secure comunication channel. +general purpose secure communication channel. Editor: Thomas S Hatch <thatch45@gmail.com> diff --git a/doc/topics/specs/salt_auth_proto_abs.rst b/doc/topics/specs/salt_auth_proto_abs.rst index e237c43ee235..d1540e3c5e16 100644 --- a/doc/topics/specs/salt_auth_proto_abs.rst +++ b/doc/topics/specs/salt_auth_proto_abs.rst @@ -2,10 +2,10 @@ Abstract on Salt Authentication and Encryption ============================================== -The Salt authentication and entcryption system uses Public Key authentication -and AES encryption to faciliate both authentication and high speed encryption. +The Salt authentication and encryption system uses Public Key authentication +and AES encryption to facilitate both authentication and high speed encryption. -The core components of this system can be seperated into a few sections, +The core components of this system can be separated into a few sections, Message Formatting, PubKey Handshake, AES key management, and encryption. Message Formatting @@ -24,7 +24,7 @@ The message itself is abstracted as a python dict in this fashion: 'load': <encrypted python pickle>} When this message is received the load can be decrypted using the shared AES -key. The 'enc' dict ket can also be "pub" for pubkey encryption, or "clear" +key. The 'enc' dict key can also be "pub" for pubkey encryption, or "clear" for passing messages in the clear. PubKey Handshake @@ -32,13 +32,13 @@ PubKey Handshake RSA Public keys are generated on the Salt master and on the Salt minion. When A salt minion establishes an initial connection to the salt master the minion -sends its public key in the clear to the salt master, allong with the id of +sends its public key in the clear to the salt master, along with the id of the minion, and the command to execute on the master, in this case "_auth": .. code-block:: python {'enc': 'clear', - 'load': + 'load': {'cmd': '_auth', 'id': <minion id>, 'pub': <minion public key>}} @@ -53,17 +53,17 @@ information on how to connect to the salt master publish interface. The returned AES key is encrypted with the minion's public key, and can therefore only be decrypted by the minion that sent out the public key. -Once the minion has authenticated and is in possesion of the revolving master -AES key (The AES key is regererated when the master restarts) then it atatches +Once the minion has authenticated and is in possession of the revolving master +AES key (The AES key is regenerated when the master restarts) then it attaches the minion subscriber to the master publisher. -All messages sent from the publisher are encypted using the revolving AES key, +All messages sent from the publisher are encrypted using the revolving AES key, in the event that the master restarts the minions will all have an invalid AES key because it has been regenerated on the master. The master will then send out a publication that the minions cannot decrypt. If the minion receives a publication that cannot be decrypted then the minion will re-authenticate, obtain the correct AES key, and decrypt the message. This means that the -AES key on the salt master can safely revolve without inturupting the minion +AES key on the salt master can safely revolve without interrupting the minion connection. Regular Communication @@ -91,7 +91,7 @@ Conclusion In the end Salt uses formatted messages with clear header data to specify how the message data is encrypted. Only uses pubkey encryption for authentication -and to securely retrive the master AES key. Then all regular communication +and to securely retrieve the master AES key. Then all regular communication is sent in AES encrypted messages. diff --git a/doc/topics/targeting/grains.rst b/doc/topics/targeting/grains.rst index cce8c1c7420d..68e371bbd894 100644 --- a/doc/topics/targeting/grains.rst +++ b/doc/topics/targeting/grains.rst @@ -25,29 +25,29 @@ Just add the option ``grains`` and pass options to it: .. code-block:: yaml grains: - roles: + roles: - webserver - memcache deployment: datacenter4 cabinet: 13 cab_u: 14-15 -Then statis data specific to your servers can be retrived via Salt, or used +Then status data specific to your servers can be retrieved via Salt, or used inside of the state system for matching. It also makes targeting, in the case of the example above, simply based on specific data about your deployment. Writing Grains ============== -Grains are easy to write. The grains interface is derived by executing all of -the "public" functions found in the modules located in the grains package or -the custom grains directory. The functions in the modules of the grains must -return a python dict, where the keys in the dict are the names of the grains and -the values are the values. +Grains are easy to write. The grains interface is derived by executing +all of the "public" functions found in the modules located in the grains +package or the custom grains directory. The functions in the modules of +the grains must return a python dict, where the keys in the dict are the +names of the grains and the values are the values. -Custom grains should be placed in a ``_grains`` directory located under your -:conf_master:`file_roots`. Before adding a grain to salt, consider what the grain -is and remember that grains need to be static data. +Custom grains should be placed in a ``_grains`` directory located under +your :conf_master:`file_roots`. Before adding a grain to salt, consider +what the grain is and remember that grains need to be static data. Examples of Grains ------------------ diff --git a/doc/topics/troubleshooting/index.rst b/doc/topics/troubleshooting/index.rst index d24fef8dffa7..bd0540635b83 100644 --- a/doc/topics/troubleshooting/index.rst +++ b/doc/topics/troubleshooting/index.rst @@ -2,7 +2,7 @@ Troubleshooting =============== -The intent of the troubleshootign secion is to introduce solutions to a +The intent of the troubleshooting section is to introduce solutions to a number of common issues encountered by users and the tools that are available to aid in developing states and salt code. @@ -23,32 +23,32 @@ Using salt-call The salt-call command was originally developed for aiding in the development of new salt modules. Since then many applications have arisen for the salt-call command that is bundled with the salt minion. These range from the original -intent of the salt-call, developmetn assistance, to gathering large amounts of +intent of the salt-call, development assistance, to gathering large amounts of data from complex calls like state.highstate. -When developing the state tree it is geenrally recommended to invoke +When developing the state tree it is generally recommended to invoke state.highstate with salt-call, this displays a great deal more information -about the highstate execution then if it is called remotely. +about the highstate execution than if it is called remotely. Too many open files =================== The salt-master needs at least 2 sockets per host that connects to it, one for -the Publisher and one for response port. Thus, large installations may upon +the Publisher and one for response port. Thus, large installations may upon scaling up the number of minions accessing a given master, encounter: 12:45:29,289 [salt.master ][INFO ] Starting Salt worker process 38 Too many open files sock != -1 (tcp_listener.cpp:335) -The solution to this would be to check the number of files allowed to be +The solution to this would be to check the number of files allowed to be opened by the user running salt-master (root by default): [root@salt-master ~]# ulimit -n 1024 And modify that value to be at least equal to the number of minions x 2. -This setting can be changed in limits.conf as the nofile value(s), +This setting can be changed in limits.conf as the nofile value(s), and activated upon new a login of the specified user. So, an environment with 1800 minions, would need 1800 x 2 = 3600 as a minimum diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index 1d76bd451ae1..4bf71164bafc 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -18,11 +18,10 @@ no tabs have crept in! Indentation =========== -The suggested -syntax for Yaml files is to use 2 spaces for indentation, but Yaml will -follow whatever indentation system that the individual file uses. Generally -2 space indentation works very well for sls files given the fact that the -represented data is uniform and not deeply nested. +The suggested syntax for Yaml files is to use 2 spaces for indentation, +but Yaml will follow whatever indentation system that the individual file +uses. Generally 2 space indentation works very well for sls files given +the fact that the represented data is uniform and not deeply nested. Nested Dicts (key-value) ------------------------ @@ -52,7 +51,7 @@ state: Notice that the spacing used is 2 spaces, and that when defining the context and defaults options there is a 4 space indent. If only a 2 space indent is used then the information will not be correctly loaded. If using double spacing -is not desireable, then a deeply nested dict can be declared with curly braces: +is not desirable, then a deeply nested dict can be declared with curly braces: .. code-block:: yaml diff --git a/doc/topics/tutorials/bootstrap_ec2.rst b/doc/topics/tutorials/bootstrap_ec2.rst index e7732fa82efa..0f937187de84 100644 --- a/doc/topics/tutorials/bootstrap_ec2.rst +++ b/doc/topics/tutorials/bootstrap_ec2.rst @@ -2,9 +2,16 @@ Boostrapping Salt on Linux EC2 with Cloud-Init ============================================== -`Salt <http://saltstack.org>`_ is a great tool for remote execution and configuration management, however you will still need to bootstrap the daemon when spinning up a new node. One option is to create and save a custom AMI, but this creates another resource to maintain and document. +`Salt <http://saltstack.org>`_ is a great tool for remote execution and +configuration management, however you will still need to bootstrap the +daemon when spinning up a new node. One option is to create and save a +custom AMI, but this creates another resource to maintain and document. -A better method for Linux machines uses Canonical's `CloudInit <https://help.ubuntu.com/community/CloudInit>`_ to run a bootstrap script during an EC2 Instance initialization. Cloud-init takes the ``user_data`` string passed into a new AWS instance and runs it in a manner similar to rc.local. The bootstrap script needs to: +A better method for Linux machines uses Canonical's `CloudInit +<https://help.ubuntu.com/community/CloudInit>`_ to run a bootstrap script +during an EC2 Instance initialization. Cloud-init takes the ``user_data`` +string passed into a new AWS instance and runs it in a manner similar to +rc.local. The bootstrap script needs to: #. Install `Salt`_ with dependencies #. Point the minion to the master @@ -24,12 +31,17 @@ Here is a sample script:: sed -i '' -e 's/#master: salt/master: [salt_master_fqdn]' /etc/salt/minion salt-minion -d -First the script adds the saltstack ppa and installs the package. Then we copy over the minion config template and tell it where to find the master. You will have to replace ``[salt_master_fqdn]`` with something that resolves to your salt master. +First the script adds the saltstack ppa and installs the package. Then +we copy over the minion config template and tell it where to find the +master. You will have to replace ``[salt_master_fqdn]`` with something +that resolves to your salt master. Used With Boto -------------- -`Boto <https://github.com/boto/boto>`_ will accept a string for user data which can be used to pass our bootstrap script. If the script is saved to a file, you can read it into a string:: +`Boto <https://github.com/boto/boto>`_ will accept a string for user data +which can be used to pass our bootstrap script. If the script is saved to +a file, you can read it into a string:: import boto @@ -47,4 +59,6 @@ Additional Notes Sometime in the future the ppa will include and install an upstart file. In the meantime, you can use the bootstrap to `build one <https://gist.github.com/1617054>`_. -It may also be useful to set the node's role during this phase. One option would be saving the node's role to a file and then using a custom grain to select it. \ No newline at end of file +It may also be useful to set the node's role during this phase. One option +would be saving the node's role to a file and then using a custom grain +to select it. From d87c8016b94dac77e9bc37e68aeccbb520270f24 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 14 Mar 2012 19:28:15 -0600 Subject: [PATCH 358/598] Add loggign to caller.py --- salt/cli/caller.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/cli/caller.py b/salt/cli/caller.py index ef9e0b3cb9ea..2d789fe46177 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -5,6 +5,7 @@ # Import python modules import sys +import logging import traceback # Import salt libs From 3a3db83012aa37dea44f728245ffbbe5374dbbcd Mon Sep 17 00:00:00 2001 From: Grier Johnson <grierj@gmail.com> Date: Wed, 14 Mar 2012 18:35:07 -0700 Subject: [PATCH 359/598] Adding in append_domain config values. Allows for a harcoded domain to be appended to minions. This helps with systems like solaris who, by default, give back the short name when you ask for socket.getfqdn() --- conf/minion.template | 5 +++++ salt/config.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/conf/minion.template b/conf/minion.template index 46b89c5bf3d0..7f1801b1a437 100644 --- a/conf/minion.template +++ b/conf/minion.template @@ -24,6 +24,11 @@ # clusters. #id: +# Append a domain to a hostname in the event that it does not exist. This is +# usefule for systems where socket.getfqdn() does not actually result in a +# FQDN (for instance, Solaris). +#append_domain: + # If the the connection to the server is interrupted, the minion will # attempt to reconnect. sub_timeout allows you to control the rate # of reconnection attempts (in seconds). To disable reconnects, set diff --git a/salt/config.py b/salt/config.py index 08ad99e0dc78..a7c2b709d6c1 100644 --- a/salt/config.py +++ b/salt/config.py @@ -42,6 +42,17 @@ def _validate_file_roots(file_roots): file_roots[env] = [] return file_roots +def _append_domain(opts): + ''' + Append a domain to the existing id if it doesn't already exist + ''' + # Domain already exists + if opts['id'].endswith(opts['append_domain']): + return opts['id'] + # Trailing dot should mean an FQDN that is terminated, leave it alone. + if opts['id'].endswith('.'): + return opts['id'] + return "{0[id]}.{0[append_domain]}".format(opts) def _read_conf_file(path): with open(path, 'r') as conf_file: @@ -164,6 +175,9 @@ def minion_config(path): if 'include' in opts: opts = include_config(opts, path) + if 'append_domain' in conf_opts: + opts['id'] = _append_domain(opts) + opts['master_ip'] = salt.utils.dns_check(opts['master']) opts['master_uri'] = 'tcp://{ip}:{port}'.format(ip=opts['master_ip'], From f0134bb83faac9c3cc012013435b71508895f8be Mon Sep 17 00:00:00 2001 From: Grier Johnson <grierj@gmail.com> Date: Wed, 14 Mar 2012 18:42:28 -0700 Subject: [PATCH 360/598] Fixing typo --- salt/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 6815d6616b54..928ea59ee481 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -119,7 +119,7 @@ def __parse(self): dest='range', action='store_true', help=('Instead of using shell globs to evaluate the target ' - 'use a range expressions to identify targets. ' + 'use a range expression to identify targets. ' 'Range expressions look like %cluster')) parser.add_option('-C', '--compound', From c810c5a02aae39382c738ec44c19089e3cab1d34 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Wed, 14 Mar 2012 19:43:45 -0600 Subject: [PATCH 361/598] Seperate out key Highstate object components This change will make extending the Highstate object and adding in a custom file client and state a viable option --- salt/state.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/salt/state.py b/salt/state.py index 30bca0a92ec3..cfee97721efe 100644 --- a/salt/state.py +++ b/salt/state.py @@ -801,17 +801,13 @@ def call_template_str(self, template): return high -class HighState(object): +class BaseHighState(object): ''' - Generate and execute the salt "High State". The High State is the - compound state derived from a group of template files stored on the - salt master or in the local cache. + The BaseHighState is the foundation of running a highstate, extend it and + add a self.state opbject of type State ''' def __init__(self, opts): - self.client = salt.fileclient.get_file_client(opts) self.opts = self.__gen_opts(opts) - self.state = State(self.opts) - self.matcher = salt.minion.Matcher(self.opts) def __gen_opts(self, opts): ''' @@ -1183,3 +1179,16 @@ def compile_low_chunks(self): if errors: return errors return chunks + + +class HighState(BaseHighState): + ''' + Generate and execute the salt "High State". The High State is the + compound state derived from a group of template files stored on the + salt master or in the local cache. + ''' + def __init__(self, opts): + self.client = salt.fileclient.get_file_client(opts) + BaseHighState.__init__(self, opts) + self.state = State(self.opts) + self.matcher = salt.minion.Matcher(self.opts) From 350e9d6d6d39f7be7bd9e8c04a7afce1593e1c27 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Wed, 14 Mar 2012 20:03:33 -0700 Subject: [PATCH 362/598] Fix var name in minion config --- salt/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/config.py b/salt/config.py index a7c2b709d6c1..dcdf0a728719 100644 --- a/salt/config.py +++ b/salt/config.py @@ -175,7 +175,7 @@ def minion_config(path): if 'include' in opts: opts = include_config(opts, path) - if 'append_domain' in conf_opts: + if 'append_domain' in opts: opts['id'] = _append_domain(opts) opts['master_ip'] = salt.utils.dns_check(opts['master']) From fde0b312612f93ee513553f52d656e21a447c342 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Wed, 14 Mar 2012 20:03:59 -0700 Subject: [PATCH 363/598] Try to support sed options across multiple platforms --- salt/modules/file.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index a852a6ddfbe7..9caa451e4042 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -11,6 +11,7 @@ import pwd import time import hashlib +import sys import salt.utils.find from salt.exceptions import SaltInvocationError @@ -504,8 +505,15 @@ def contains(path, text, limit='', escape=False): if not os.path.exists(path): return False + if 'linux' in sys.platform: + options = '-n -r -e' + elif 'darwin' in sys.platform: + options = '-n -E -e' + else: + options = '-n -e' + result = __salt__['file.sed'](path, text, '&', limit=limit, backup='', - options='-n -r -e', flags='gp', escape_all=escape) + options=options, flags='gp', escape_all=escape) return bool(result) From 564246ed6eb4440717c938155bb9d376eb829eb3 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Thu, 15 Mar 2012 08:50:27 -0700 Subject: [PATCH 364/598] Add warning for yaml renderers when a key is overridden, #631 --- salt/renderers/utils.py | 45 ++++++++++++++++++++++++++++++++++++ salt/renderers/yaml_jinja.py | 23 ++++++++++-------- salt/renderers/yaml_mako.py | 18 +++++++++------ 3 files changed, 69 insertions(+), 17 deletions(-) create mode 100644 salt/renderers/utils.py diff --git a/salt/renderers/utils.py b/salt/renderers/utils.py new file mode 100644 index 000000000000..534187661fd3 --- /dev/null +++ b/salt/renderers/utils.py @@ -0,0 +1,45 @@ +import warnings + +# Import thirt party modules +import yaml +from yaml.nodes import MappingNode +from yaml.constructor import ConstructorError +try: + yaml.Loader = yaml.CLoader + yaml.Dumper = yaml.CDumper +except: + pass + +load = yaml.load + +class CustomeConstructor(yaml.constructor.SafeConstructor): + def construct_mapping(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + mapping = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + try: + hash(key) + except TypeError, exc: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unacceptable key (%s)" % exc, key_node.start_mark) + value = self.construct_object(value_node, deep=deep) + if key in mapping: + warnings.warn("Duplicate Key: '{0}'".format(key)) + mapping[key] = value + return mapping + + +class CustomLoader(yaml.reader.Reader, yaml.scanner.Scanner, yaml.parser.Parser, + yaml.composer.Composer, CustomeConstructor, yaml.resolver.Resolver): + def __init__(self, stream): + yaml.reader.Reader.__init__(self, stream) + yaml.scanner.Scanner.__init__(self) + yaml.parser.Parser.__init__(self) + yaml.composer.Composer.__init__(self) + CustomeConstructor.__init__(self) + yaml.resolver.Resolver.__init__(self) + diff --git a/salt/renderers/yaml_jinja.py b/salt/renderers/yaml_jinja.py index 6d19a16623ef..9276f0374a67 100644 --- a/salt/renderers/yaml_jinja.py +++ b/salt/renderers/yaml_jinja.py @@ -7,18 +7,16 @@ # Import Python Modules import os - -# Import thirt party modules -import yaml -try: - yaml.Loader = yaml.CLoader - yaml.Dumper = yaml.CDumper -except: - pass +import logging +import warnings # Import Salt libs from salt.utils.jinja import get_template +from salt.renderers.utils import CustomLoader, load + +log = logging.getLogger(__name__) + def render(template_file, env='', sls=''): ''' Render the data passing the functions and grains into the rendering system @@ -36,5 +34,10 @@ def render(template_file, env='', sls=''): template = get_template(template_file, __opts__, env) yaml_data = template.render(**passthrough) - - return yaml.safe_load(yaml_data) + with warnings.catch_warnings(record=True) as warn_list: + data = load(yaml_data, Loader=CustomLoader) + if len(warn_list) > 0: + for item in warn_list: + log.warn("{warn} found in {file_}".format( + warn=item.message, file_=template_file)) + return data diff --git a/salt/renderers/yaml_mako.py b/salt/renderers/yaml_mako.py index 609532798175..6076b612cd07 100644 --- a/salt/renderers/yaml_mako.py +++ b/salt/renderers/yaml_mako.py @@ -10,12 +10,8 @@ # Import Third Party libs from mako.template import Template -import yaml -try: - yaml.Loader = yaml.CLoader - yaml.Dumper = yaml.CDumper -except: - pass + +from salt.renderers.utils import CustomLoader, load def render(template, env='', sls=''): @@ -35,4 +31,12 @@ def render(template, env='', sls=''): template = Template(open(template, 'r').read()) yaml_data = template.render(**passthrough) - return yaml.safe_load(yaml_data) + + yaml_data = template.render(**passthrough) + with warnings.catch_warnings(record=True) as warn_list: + data = load(yaml_data, Loader=CustomLoader) + if len(warn_list) > 0: + for item in warn_list: + log.warn("{warn} found in {file_}".format( + warn=item.message, file_=template_file)) + return data From 630f96e175e16f6e71daceaeefbdfee92e53f94d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 15 Mar 2012 09:56:46 -0600 Subject: [PATCH 365/598] fix caps in jobs.active --- salt/runners/jobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/runners/jobs.py b/salt/runners/jobs.py index 7ce378470b40..e03492722ebc 100644 --- a/salt/runners/jobs.py +++ b/salt/runners/jobs.py @@ -35,7 +35,7 @@ def active(): 'Target': job['tgt'], 'Target-type': job['tgt_type']} else: - ret[job['jid']]['running'].append({minion: job['pid']}) + ret[job['jid']]['Running'].append({minion: job['pid']}) if os.path.isdir(job_dir): for jid in os.listdir(job_dir): if not jid in ret: @@ -47,7 +47,7 @@ def active(): if minion.startswith('.'): continue if os.path.exists(os.path.join(jid_dir, minion)): - ret[jid]['returned'].append(minion) + ret[jid]['Returned'].append(minion) print yaml.dump(ret) From 91a22fb98de0be51e2b202d074fa4db7c561606c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 15 Mar 2012 10:03:20 -0600 Subject: [PATCH 366/598] Add MasterState class to state.py --- salt/state.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/salt/state.py b/salt/state.py index cfee97721efe..7584f268552c 100644 --- a/salt/state.py +++ b/salt/state.py @@ -801,6 +801,29 @@ def call_template_str(self, template): return high +class MasterState(State): + ''' + Create a State object for master side compiling + ''' + def __init__(self, opts, minion): + State.__init__(self, opts) + self.minion = minion + + def load_modules(self, data=None): + ''' + Load the modules into the state + ''' + log.info('Loading fresh modules for state activity') + # Load a modified client interface that looks like the interface used + # from the minion, but uses remote execution + # + self.functions = salt.client.FunctionWrapper(self.opts, self.minion) + # Load the states, but they should not be used in this class apart + # from inspection + self.states = salt.loader.states(self.opts, self.functions) + self.rend = salt.loader.render(self.opts, self.functions) + + class BaseHighState(object): ''' The BaseHighState is the foundation of running a highstate, extend it and From a5267584b41a43dbba8e66f115bedde523aa86a4 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 15 Mar 2012 10:23:13 -0700 Subject: [PATCH 367/598] Make Tom happy Thanks @dcolish for pointing this out. --- salt/utils/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index ed463daa471e..f5010769c866 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -3,6 +3,7 @@ ''' import os +import imp import sys import socket import logging @@ -284,7 +285,7 @@ def required_module_list(docstring=None): modules = mod_list[0].replace(txt, '').split(', ') for mod in modules: try: - __import__(mod) + imp.find_module(mod) except ImportError: ret.append(mod) return ret From 60edc32b3a994adcfba4f7b6b4b01028a975fa1a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 15 Mar 2012 11:54:27 -0600 Subject: [PATCH 368/598] Add FunctionWrapper to the client --- salt/client.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/salt/client.py b/salt/client.py index 583a46e9047e..c1027928d6b6 100644 --- a/salt/client.py +++ b/salt/client.py @@ -695,3 +695,47 @@ def pub(self, tgt, fun, arg=(), expr_form='glob', return {'jid': '0', 'minions': []} return {'jid': payload['load']['jid'], 'minions': minions} + + +class FunctionWrapper(dict): + ''' + Create a function wrapper that looks like the functions dict on the minion + but invoked commands on the minion via a LocalClient. + + This allows SLS files to be loaded with an object that calls down to the + minion when the salt functions dict is referenced. + ''' + def __init__(self, opts, minion): + self.opts = opts + self.minion = minion + self.local = LocalClient(self.opts['conf_file']) + self.functions = self.__load_functions() + + def __missing__(self, key): + ''' + Since the function key is missing, wrap this call to a command to the + minion of said key if it is available in the self.functions set + ''' + if not key in self.functions: + raise KeyError + return self.run_key(key) + + def __load_functions(self): + ''' + Find out what functions are available on the minion + ''' + return set(self.local.cmd(self.minion, 'sys.functions')) + + def run_key(self, key): + ''' + Return a function that executes the arguments passed via the local + client + ''' + def func(*args, **kwargs): + ''' + Run a remote call + ''' + args = list(args) + for _key, _val in kwargs: + args.append('{0}={1}'.format(_key, _val)) + return self.local.cmd(self.minion, key, args) From 531770491eda526c136a1d6a02fce586ac628fc9 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 15 Mar 2012 12:24:38 -0600 Subject: [PATCH 369/598] Add MasterHighState class for master side compiling --- salt/state.py | 60 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/salt/state.py b/salt/state.py index 7584f268552c..9a39407e9d79 100644 --- a/salt/state.py +++ b/salt/state.py @@ -801,29 +801,6 @@ def call_template_str(self, template): return high -class MasterState(State): - ''' - Create a State object for master side compiling - ''' - def __init__(self, opts, minion): - State.__init__(self, opts) - self.minion = minion - - def load_modules(self, data=None): - ''' - Load the modules into the state - ''' - log.info('Loading fresh modules for state activity') - # Load a modified client interface that looks like the interface used - # from the minion, but uses remote execution - # - self.functions = salt.client.FunctionWrapper(self.opts, self.minion) - # Load the states, but they should not be used in this class apart - # from inspection - self.states = salt.loader.states(self.opts, self.functions) - self.rend = salt.loader.render(self.opts, self.functions) - - class BaseHighState(object): ''' The BaseHighState is the foundation of running a highstate, extend it and @@ -1215,3 +1192,40 @@ def __init__(self, opts): BaseHighState.__init__(self, opts) self.state = State(self.opts) self.matcher = salt.minion.Matcher(self.opts) + + +class MasterState(State): + ''' + Create a State object for master side compiling + ''' + def __init__(self, opts, minion): + State.__init__(self, opts) + self.minion = minion + + def load_modules(self, data=None): + ''' + Load the modules into the state + ''' + log.info('Loading fresh modules for state activity') + # Load a modified client interface that looks like the interface used + # from the minion, but uses remote execution + # + self.functions = salt.client.FunctionWrapper(self.opts, self.minion) + # Load the states, but they should not be used in this class apart + # from inspection + self.states = salt.loader.states(self.opts, self.functions) + self.rend = salt.loader.render(self.opts, self.functions) + + +class MasterHighState(BaseHighState): + ''' + Execute highstate compilation from the master + ''' + def __init__(self, opts, minion): + # Force the fileclient to be local + opts = copy.deepcopy(opts) + opts['file_client'] = 'local' + self.client = salt.fileclient.get_file_client(opts) + BaseHighState.__init__(self, opts) + self.state = MasterState(self.opts, minion) + self.matcher = salt.minion.Matcher(self.opts) From 4ceacf69128617c5a261af8beb111ebbefcc46b2 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Thu, 15 Mar 2012 13:00:06 -0700 Subject: [PATCH 370/598] Move yaml utilities to salt.utils, #631 --- salt/renderers/yaml_jinja.py | 3 +-- salt/renderers/yaml_mako.py | 2 +- salt/{renderers/utils.py => utils/yaml.py} | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) rename salt/{renderers/utils.py => utils/yaml.py} (97%) diff --git a/salt/renderers/yaml_jinja.py b/salt/renderers/yaml_jinja.py index 9276f0374a67..bd3661956c4d 100644 --- a/salt/renderers/yaml_jinja.py +++ b/salt/renderers/yaml_jinja.py @@ -12,8 +12,7 @@ # Import Salt libs from salt.utils.jinja import get_template - -from salt.renderers.utils import CustomLoader, load +from salt.utils.yaml import CustomLoader, load log = logging.getLogger(__name__) diff --git a/salt/renderers/yaml_mako.py b/salt/renderers/yaml_mako.py index 6076b612cd07..7db1ead73d5d 100644 --- a/salt/renderers/yaml_mako.py +++ b/salt/renderers/yaml_mako.py @@ -11,7 +11,7 @@ # Import Third Party libs from mako.template import Template -from salt.renderers.utils import CustomLoader, load +from salt.utils.yaml import CustomLoader, load def render(template, env='', sls=''): diff --git a/salt/renderers/utils.py b/salt/utils/yaml.py similarity index 97% rename from salt/renderers/utils.py rename to salt/utils/yaml.py index 534187661fd3..c08248095b1e 100644 --- a/salt/renderers/utils.py +++ b/salt/utils/yaml.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import warnings # Import thirt party modules From d2a6b3b6dff4017adb3a2eee06f2ed343f35c049 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 15 Mar 2012 19:18:12 -0700 Subject: [PATCH 371/598] Lazy load _winreg to prevent any errors --- salt/modules/reg.py | 51 ++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/salt/modules/reg.py b/salt/modules/reg.py index c457fccfc0df..c64937788134 100644 --- a/salt/modules/reg.py +++ b/salt/modules/reg.py @@ -4,6 +4,9 @@ Required python modules: _winreg ''' +# TODO: Figure out the exceptions _winreg can raise and properly catch +# them instead of a bare except that catches any exception at all + try: import _winreg has_windows_modules = True @@ -11,6 +14,25 @@ has_windows_modules = False import salt.utils +from salt.exceptions import CommandExecutionError + +class Registry(object): + ''' + Delay '_winreg' usage until this module is used + ''' + def __init__(self): + hkeys = { + "HKEY_USERS": _winreg.HKEY_USERS, + "HKEY_CURRENT_USER": _winreg.HKEY_CURRENT_USER, + "HKEY_LOCAL_MACHINE": _winreg.HKEY_LOCAL_MACHINE, + } + def __getattr__(self, k): + try: + return self.hkeys[k] + except KeyError: + msg = 'No hkey named \'{0}. Try one of {1}\'' + hkeys = ', '.join(self.hkeys) + raise CommandExecutionError(msg.format(k, hkeys)) def __virtual__(): ''' @@ -22,13 +44,6 @@ def __virtual__(): log.warn(salt.utils.required_modules_error(__file__, __doc__)) return False - -hkeys = {'HKEY_CURRENT_USER': _winreg.HKEY_CURRENT_USER, - 'HKEY_LOCAL_MACHINE': _winreg.HKEY_LOCAL_MACHINE, - 'HKEY_USERS': _winreg.HKEY_USERS, - } - - def read_key(hkey, path, key): ''' Read registry key value @@ -37,7 +52,9 @@ def read_key(hkey, path, key): salt '*' reg.read_key HKEY_LOCAL_MACHINE 'SOFTWARE\\Salt' 'version' ''' - hkey2 = hkeys[hkey] + + registry = Registry() + hkey2 = getattr(registry, hkey) fullpath = '\\\\'.join([path, key]) try: handle = _winreg.OpenKey(hkey2, fullpath, 0, _winreg.KEY_READ) @@ -54,8 +71,10 @@ def set_key(hkey, path, key, value): salt '*' reg.set_key HKEY_CURRENT_USER 'SOFTWARE\\Salt' 'version' '0.97' ''' - hkey2 = hkeys[hkey] + registry = Registry() + hkey2 = getattr(registry, hkey) fullpath = '\\\\'.join([path, key]) + try: handle = _winreg.OpenKey(hkey2, fullpath, 0, _winreg.KEY_ALL_ACCESS) _winreg.SetValueEx(handle, key, 0, _winreg.REG_SZ, value) @@ -65,7 +84,7 @@ def set_key(hkey, path, key, value): handle = _winreg.CreateKey(hkey2, fullpath) _winreg.SetValueEx(handle, key, 0, _winreg.REG_SZ, value) _winreg.CloseKey(handle) - return True + return True def create_key(hkey, path, key, value=None): @@ -76,8 +95,10 @@ def create_key(hkey, path, key, value=None): salt '*' reg.create_key HKEY_CURRENT_USER 'SOFTWARE\\Salt' 'version' '0.97' ''' - hkey2 = hkeys[hkey] + registry = Registry() + hkey2 = getattr(registry, hkey) fullpath = '\\\\'.join([path, key]) + try: handle = _winreg.OpenKey(hkey2, fullpath, 0, _winreg.KEY_ALL_ACCESS) _winreg.CloseKey(handle) @@ -87,7 +108,7 @@ def create_key(hkey, path, key, value=None): if value: _winreg.SetValueEx(handle, key, 0, _winreg.REG_SZ, value) _winreg.CloseKey(handle) - return True + return True def delete_key(hkey, path, key): @@ -100,7 +121,9 @@ def delete_key(hkey, path, key): salt '*' reg.delete_key HKEY_CURRENT_USER 'SOFTWARE\\Salt' 'version' ''' - hkey2 = hkeys[hkey] + registry = Registry() + hkey2 = getattr(registry, hkey) + try: handle = _winreg.OpenKey(hkey2, path, 0, _winreg.KEY_ALL_ACCESS) _winreg.DeleteKeyEx(handle, key) @@ -108,4 +131,4 @@ def delete_key(hkey, path, key): return True except: _winreg.CloseKey(handle) - return True + return True From db476a1e5893a4fc86b871488d0800cb4b3af345 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 15 Mar 2012 19:31:46 -0700 Subject: [PATCH 372/598] Don't complain about missing libvirt for virt --- salt/modules/virt.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index d2645c54cd5d..900b1d66e163 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -1,17 +1,23 @@ ''' Work with virtual machines managed by libvirt + +Required python modules: libvirt ''' # Special Thanks to Michael Dehann, many of the concepts, and a few structures # of his in the virt func module have been used -from xml.dom import minidom -from salt.exceptions import CommandExecutionError -import StringIO import os import shutil +import StringIO import subprocess +from xml.dom import minidom +from salt.exceptions import CommandExecutionError -import libvirt +try: + import libvirt + has_libvirt = True +except ImportError: + has_libvirt = False # Import Third Party Libs import yaml @@ -26,6 +32,12 @@ 6: "crashed"} +def __virtual__(): + if not has_libvirt: + return False + return 'virt' + + def __get_conn(): ''' Detects what type of dom this node is and attempts to connect to the @@ -572,7 +584,7 @@ def is_xen_hyper(): if __grains__['virtual_subtype'] != 'Xen Dom0': return False except KeyError: - # virtual_subtype isn't set everywhere. + # virtual_subtype isn't set everywhere. return False try: if 'xen_' not in open('/proc/modules').read(): From 9f8a778f369eaf8c963052b69e515022ce01b233 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 15 Mar 2012 19:34:27 -0700 Subject: [PATCH 373/598] Don't complain about missing libvirt for kvm_hyper --- salt/modules/kvm_hyper.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/modules/kvm_hyper.py b/salt/modules/kvm_hyper.py index 815de126f5f7..793844a51e07 100644 --- a/salt/modules/kvm_hyper.py +++ b/salt/modules/kvm_hyper.py @@ -1,6 +1,8 @@ ''' Provide the hyper module for kvm hypervisors. This is the interface used to interact with kvm on behalf of the salt-virt interface + +Required python modules: libvirt ''' # This is a test interface for the salt-virt system. The api in this file is @@ -8,14 +10,18 @@ # Import Python Libs -from xml.dom import minidom -import StringIO import os import shutil +import StringIO import subprocess +from xml.dom import minidom # Import libvirt -import libvirt +try: + import libvirt + has_libvirt = True +except ImportError: + has_libvirt = True # Import Third party modules import yaml @@ -40,6 +46,8 @@ def __virtual__(): return False if 'kvm_' not in open('/proc/modules').read(): return False + if not has_libvirt: + return False try: libvirt_conn = libvirt.open('qemu:///system') libvirt_conn.close() From c45e4e2a677d12636b874f513931c2e7e33e5546 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 15 Mar 2012 19:40:00 -0700 Subject: [PATCH 374/598] Don't complain about missing pycassa --- salt/returners/cassandra_return.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/salt/returners/cassandra_return.py b/salt/returners/cassandra_return.py index 971640fe40c6..e2e33bb30054 100644 --- a/salt/returners/cassandra_return.py +++ b/salt/returners/cassandra_return.py @@ -1,7 +1,7 @@ ''' -Return data to a Cassandra ColumFamily +Return data to a Cassandra ColumnFamily -Here's an example Keyspace/ColumnFamily setup that works with this +Here's an example Keyspace / ColumnFamily setup that works with this returner:: create keyspace salt; @@ -10,10 +10,22 @@ with key_validation_class='UTF8Type' and comparator='UTF8Type' and default_validation_class='UTF8Type'; + +Required python modules: pycassa ''' import logging -import pycassa + +try: + import pycassa + has_pycassa = True +except ImportError + has_pycassa = False + +def __virtual__(): + if not has_pycassa: + return False + return 'cassandra' log = logging.getLogger(__name__) From 51034a8d6de4674efbdf1f093aa199726fab542c Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 15 Mar 2012 19:43:54 -0700 Subject: [PATCH 375/598] Don't complain about missing pymongo --- salt/returners/mongo_return.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/salt/returners/mongo_return.py b/salt/returners/mongo_return.py index db98104c2183..0aa19eb4ef82 100644 --- a/salt/returners/mongo_return.py +++ b/salt/returners/mongo_return.py @@ -2,10 +2,17 @@ Return data to a mongodb server This is the default interface for returning data for the butter statd subsytem + +Required python modules: pymongo ''' import logging -import pymongo + +try: + import pymongo + has_pymongo = True +except ImportError + has_pymongo = False log = logging.getLogger(__name__) @@ -17,6 +24,12 @@ 'mongo.user': ''} +def __virtual__(): + if not has_pymongo: + return False + return 'mongo_return' + + def returner(ret): ''' Return data to a mongodb server From 338d7d3cf83b57105fd23d1e68b761f87633d4f0 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 15 Mar 2012 19:48:42 -0700 Subject: [PATCH 376/598] Don't complain about missing redis Also fix a few other syntax errors introduced previously --- salt/returners/cassandra_return.py | 14 ++++++++------ salt/returners/mongo_return.py | 2 +- salt/returners/redis_return.py | 15 ++++++++++++++- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/salt/returners/cassandra_return.py b/salt/returners/cassandra_return.py index e2e33bb30054..cd2d941562bd 100644 --- a/salt/returners/cassandra_return.py +++ b/salt/returners/cassandra_return.py @@ -19,14 +19,9 @@ try: import pycassa has_pycassa = True -except ImportError +except ImportError: has_pycassa = False -def __virtual__(): - if not has_pycassa: - return False - return 'cassandra' - log = logging.getLogger(__name__) __opts__ = {'cassandra.servers': ['localhost:9160'], @@ -34,6 +29,13 @@ def __virtual__(): 'cassandra.column_family': 'returns', 'cassandra.consistency_level': 'ONE'} + +def __virtual__(): + if not has_pycassa: + return False + return 'cassandra' + + def returner(ret): ''' Return data to a Cassandra ColumnFamily diff --git a/salt/returners/mongo_return.py b/salt/returners/mongo_return.py index 0aa19eb4ef82..7d21877d31a0 100644 --- a/salt/returners/mongo_return.py +++ b/salt/returners/mongo_return.py @@ -11,7 +11,7 @@ try: import pymongo has_pymongo = True -except ImportError +except ImportError: has_pymongo = False diff --git a/salt/returners/redis_return.py b/salt/returners/redis_return.py index 351727b923e3..5f6c7dcfabdb 100644 --- a/salt/returners/redis_return.py +++ b/salt/returners/redis_return.py @@ -2,16 +2,29 @@ Return data to a redis server This is a VERY simple example for pushing data to a redis server and is not necessarily intended as a usable interface. + +Required python modules: redis ''' import json -import redis + +try: + import redis + has_redis = True +except ImportError: + has_redis = False __opts__ = {'redis.db': '0', 'redis.host': 'mcp', 'redis.port': 6379} +def __virtual__(): + if not has_redis: + return False + return 'redis_return' + + def returner(ret): ''' Return data to a redis data store From 30493fb4c2e7964036bda333660d1e645b95fef3 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 15 Mar 2012 20:37:58 -0700 Subject: [PATCH 377/598] Strip the trailing newline / whitespace from cmd.* This was the most frequent problem we ran into whenever a user would try something like: __salt__['cmd.run']('foo') inside of a state file or template. Lets ease their frustration There is no way to selectively disable this in the various cmd module commands, but that can easily be added later on if it is ever deemend necessary. --- salt/modules/cmdmod.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 7defc5f2e9a7..fbbcde95dc89 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -46,7 +46,8 @@ def _run(cmd, runas=None, with_env=True, shell=DEFAULT_SHELL, - env=()): + env=(), + rstrip=True): ''' Do the DRY thing and only call subprocess.Popen() once ''' @@ -105,6 +106,16 @@ def _run(cmd, proc = subprocess.Popen(cmd, **kwargs) out = proc.communicate() + + if rstrip: + # Cast out to a list as proc.communicate() returns a tuple + out = list(out) + if out[0]: + out[0] = out[0].rstrip() + # None lacks a rstrip() method + if out[1]: + out[1] = out[1].rstrip() + ret['stdout'] = out[0] ret['stderr'] = out[1] ret['pid'] = proc.pid From 6032b2a2db6381513e3db7622e65367cec85b9db Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 15 Mar 2012 21:42:47 -0600 Subject: [PATCH 378/598] Add master_roots option --- salt/config.py | 3 +++ salt/state.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/salt/config.py b/salt/config.py index dcdf0a728719..aaa863490ba9 100644 --- a/salt/config.py +++ b/salt/config.py @@ -217,6 +217,9 @@ def master_config(path): 'file_roots': { 'base': ['/srv/salt'], }, + 'master_roots': { + 'base': ['/srv/salt-master'], + }, 'pillar_roots': { 'base': ['/srv/pillar'], }, diff --git a/salt/state.py b/salt/state.py index 9a39407e9d79..bbed256ee0ac 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1225,7 +1225,9 @@ def __init__(self, opts, minion): # Force the fileclient to be local opts = copy.deepcopy(opts) opts['file_client'] = 'local' + opts['file_roots'] = opts['master_roots'] self.client = salt.fileclient.get_file_client(opts) BaseHighState.__init__(self, opts) + # Use the master state object self.state = MasterState(self.opts, minion) self.matcher = salt.minion.Matcher(self.opts) From 8b86bbe0c9961533669e94e719a17431055cad6f Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 15 Mar 2012 20:44:10 -0700 Subject: [PATCH 379/598] Add service.reload to fix #907 --- salt/modules/service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/modules/service.py b/salt/modules/service.py index 103a6eea0bf2..efe75f8afc36 100644 --- a/salt/modules/service.py +++ b/salt/modules/service.py @@ -90,3 +90,15 @@ def status(name, sig=None): __grains__, sig) return __salt__['cmd.run'](cmd).strip() + +def reload(name): + ''' + Restart the named service + + CLI Example:: + + salt '*' service.reload <service name> + ''' + cmd = os.path.join(grainmap[__grains__['os']], + name + ' reload') + return not __salt__['cmd.retcode'](cmd) From 42d260846d7e46a2b9303408f23226c7a76fee18 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 15 Mar 2012 22:34:14 -0600 Subject: [PATCH 380/598] Add gateway for reading in the master highstate --- salt/master.py | 13 +++++++++++++ salt/state.py | 12 +++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/salt/master.py b/salt/master.py index ed6610ea2124..2f8e8fd4a547 100644 --- a/salt/master.py +++ b/salt/master.py @@ -29,6 +29,7 @@ import salt.client import salt.payload import salt.pillar +import salt.state log = logging.getLogger(__name__) @@ -553,6 +554,18 @@ def _pillar(self, load): load['env']) return pillar.compile_pillar() + def _master_state(self, load): + ''' + Call the master to compile a master side highstate + ''' + if 'id' not in load or 'grains' not in load or 'env' not in load: + return False + return salt.state.master_compile( + self.opts, + load['grains'], + load['id'], + load['env']) + def _return(self, load): ''' Handle the return data sent from the minions diff --git a/salt/state.py b/salt/state.py index bbed256ee0ac..0f25a5a6f001 100644 --- a/salt/state.py +++ b/salt/state.py @@ -124,6 +124,13 @@ def format_log(ret): log.info(str(ret)) +def master_compile(opts, grains, id_, env): + ''' + Compile the master side low state data, and build the hidden state file + ''' + st_ = MasterHighState(opts, grains, id_, env) + return st_.compile_highstate() + def ishashable(obj): try: hash(obj) @@ -1221,11 +1228,14 @@ class MasterHighState(BaseHighState): ''' Execute highstate compilation from the master ''' - def __init__(self, opts, minion): + def __init__(self, opts, grains, id_, env=None): # Force the fileclient to be local opts = copy.deepcopy(opts) opts['file_client'] = 'local' opts['file_roots'] = opts['master_roots'] + opts['id'] = minion + opts['grains'] = grains + opts['environment'] = env self.client = salt.fileclient.get_file_client(opts) BaseHighState.__init__(self, opts) # Use the master state object From 439331427256d91de56fc6883abdee9c96fefd49 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 15 Mar 2012 22:57:51 -0600 Subject: [PATCH 381/598] Add minion side connections for master compiling of states --- salt/modules/state.py | 11 +++++++++++ salt/state.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/salt/modules/state.py b/salt/modules/state.py index 79d331818fc6..71db4d850321 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -128,3 +128,14 @@ def show_lowstate(): ''' st_ = salt.state.HighState(__opts__) return st_.compile_low_chunks() + +def show_masterstate(): + ''' + Display the data gathered from the master compiled state + + CLI Example:: + + salt '*' state.show_masterstate + ''' + st_ = salt.state.RemoteHighState(__opts__, __grains__) + return st_.compile_master() diff --git a/salt/state.py b/salt/state.py index 0f25a5a6f001..e612dbea4b8a 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1241,3 +1241,38 @@ def __init__(self, opts, grains, id_, env=None): # Use the master state object self.state = MasterState(self.opts, minion) self.matcher = salt.minion.Matcher(self.opts) + + +class RemoteHighState(object): + ''' + Manage gathering the data from the master + ''' + def __init__(self, opts, grains): + self.opts = opts + self.grains = grains + self.id_ = opts['id'] + self.serial = salt.payload.Serial(self.opts) + self.auth = salt.crypt.SAuth(opts) + self.socket = self.__get_socket() + + def __get_socket(self): + ''' + Return the zeromq socket to use + ''' + context = zmq.Context() + socket = context.socket(zmq.REQ) + socket.connect(self.opts['master_uri']) + return socket + + def compile_master(self): + ''' + Return the state data from the master + ''' + payload = {'enc': 'aes'} + load = {'id': self.id_, + 'grains': self.grains, + 'env': self.opts['environment'], + 'cmd': '_master_state'} + payload['load'] = self.auth.crypticle.dumps(load) + self.socket.send(self.serial.dumps(payload)) + return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) From 9bca52ac1a8d5a0223fafcd7f9136403bb19f6f7 Mon Sep 17 00:00:00 2001 From: root <root@salt-minion.(none)> Date: Fri, 16 Mar 2012 05:17:51 +0000 Subject: [PATCH 382/598] refactor of pip --- salt/modules/pip.py | 175 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 156 insertions(+), 19 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index 43e7f7f22be3..41e1c7d84d0e 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -17,35 +17,172 @@ def _get_pip_bin(pip, env): else: return pip if pip else __opts__['pip_bin'] -def install(env='', requirements='', pkgs='', pip_bin=''): +def install(packages, + bin_env=None, + log=None, + proxy=None, + timeout=None, + editable=None, + find_links=None, + index_url=None, + extra_index_url=None, + no_index=False, + mirrors=None, + build=None, + target=None, + download=None, + download_cache=None, + source=None, + upgrade=False, + force_reinstall=False, + ignore_installed=False, + no_deps=False, + no_install=False, + no_download=False, + install_options=None): ''' Install packages with pip Install packages individually or from a pip requirements file. Install packages globally or to a virtualenv. - env : None - The path to a virtualenv that pip should install to. This option takes - precendence over the ``pip_bin`` argument. - requirements : None - The path to a pip requirements file to install from - pkgs : None - A list of space-separated packages to install - pip_bin : 'pip' - The name (and optionally path) of the pip command to call. This option - will be ignored if the ``env`` argument is given since it will default - to the pip that is installed in the virtualenv. This option can also be - set in the minion config file as ``pip.pip_bin``. + packages + package(s) or requirements file + bin_env + path to pip bin or path to virtualenv. If doing a system install, + and want to use a specific pip bin (pip-2.7, pip-2.6, etc..) just + specify the pip bin you want. + If installing into a virtualenv, just use the path to the virtualenv + (/home/code/path/to/virtualenv/) CLI Example:: - salt '*' pip.install /var/www/myvirtualenv.com \\ - /path/to/requirements.txt + salt '*' pip.install <package name>,<package2 name> + + salt '*' pip.install /path/to/requirements.txt + + salt '*' pip.install <package name> /path/to/virtualenv + + salt '*' pip.install <package name> /path/to/pip_bin ''' - cmd = '{pip_bin} install {reqs} {pkgs}'.format( - pip_bin=_get_pip_bin(pip_bin, env), - reqs='-r {0}'.format(requirements) if requirements else '', - pkgs=pkgs) + + packages = packages.split(",") + + if len(packages) == 1 and packages[0].endswith('.txt'): + # we are installing a requirements file + pkg = "-r {reqs}".format(reqs=packages[0]) + else: + # we have a list of packages to install + pkg = ' '.join(packages) + + if not bin_env: + pip_bin = 'pip' + else: + # try to get pip bin from env + if os.path.exists(os.path.join(bin_env, 'bin', 'pip')): + pip_bin = os.path.join(bin_env, 'bin', 'pip') + else: + pip_bin = bin_env + + cmd = '{pip_bin} install {pkg}'.format( + pip_bin=pip_bin, + pkg=pkg) + + if log: + try: + # TODO make this check if writeable + os.path.exists(log) + except IOError: + raise IOError("'%s' is not writeable" % log) + cmd = '{cmd} --{log}'.format( + cmd=cmd, log=log) + + if proxy: + cmd = '{cmd} --proxy={proxy}'.format( + cmd=cmd, proxy=proxy) + + if timeout: + try: + int(timeout) + except ValueError: + raise ValueError("'%s' is not a valid integer base 10.") + cmd = '{cmd} --timeout={timeout}'.format( + cmd=cmd, timeout=timeout) + + if editable: + if editable.find('egg') == -1: + raise Exception('You must specify an egg for this editable') + cmd = '{cmd} --editable={editable}'.format( + cmd=cmd, editable=editable) + + if find_links: + if not find_links.startswith("http://"): + raise Exception("'%s' must be a valid url" % find_links) + cmd = '{cmd} --find_links={find_links}'.format( + cmd=cmd, find_links=find_links) + + if index_url: + if not index_url.startswith("http://"): + raise Exception("'%s' must be a valid url" % index_url) + cmd = '{cmd} --index_url={index_url}'.format( + cmd=cmd, index_url=index_url) + + if extra_index_url: + if not extra_index_url.startswith("http://"): + raise Exception("'%s' must be a valid url" % extra_index_url) + cmd = '{cmd} --extra_index_url={extra_index_url}'.format( + cmd=cmd, extra_index_url=extra_index_url) + + if no_index: + cmd = '{cmd} --no-index'.format(cmd=cmd) + + if mirrors: + if not mirrors.startswith("http://"): + raise Exception("'%s' must be a valid url" % mirrors) + cmd = '{cmd} --use-mirrors --mirrors={mirrors}'.format( + cmd=cmd, mirrors=mirrors) + + if build: + cmd = '{cmd} --build={build}'.format( + cmd=cmd, build=build) + + if target: + cmd = '{cmd} --target={target}'.format( + cmd=cmd, target=target) + + if download: + cmd = '{cmd} --download={download}'.format( + cmd=cmd, download=download) + + if download_cache: + cmd = '{cmd} --download_cache={download_cache}'.format( + cmd=cmd, download_cache=download_cache) + + if source: + cmd = '{cmd} --source={source}'.format( + cmd=cmd, source=source) + + if upgrade: + cmd = '{cmd} --upgrade'.format(cmd=cmd) + + if force_reinstall: + cmd = '{cmd} --force-reinstall'.format(cmd=cmd) + + if ignore_installed: + cmd = '{cmd} --ignore-installed'.format(cmd=cmd) + + if no_deps: + cmd = '{cmd} --no-deps'.format(cmd=cmd) + + if no_install: + cmd = '{cmd} --no-install'.format(cmd=cmd) + + if no_download: + cmd = '{cmd} --no-download'.format(cmd=cmd) + + if install_options: + cmd = '{cmd} --install-options={install_options}'.format( + cmd=cmd, install_options=install_options) return __salt__['cmd.run'](cmd) From 46ce403463be3319cc504342d07b6c11e1394ff2 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 15 Mar 2012 23:23:05 -0600 Subject: [PATCH 383/598] Initial support for compiling states on the master is in It is far from done, but states can now be compiled on the master and the highstate returned to the minion --- salt/state.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/state.py b/salt/state.py index e612dbea4b8a..ee4b20695060 100644 --- a/salt/state.py +++ b/salt/state.py @@ -19,7 +19,10 @@ import logging import collections -# Import Salt Libs +# Import Third Party libs +import zmq + +# Import Salt libs import salt.utils import salt.loader import salt.minion @@ -1207,7 +1210,6 @@ class MasterState(State): ''' def __init__(self, opts, minion): State.__init__(self, opts) - self.minion = minion def load_modules(self, data=None): ''' @@ -1217,7 +1219,10 @@ def load_modules(self, data=None): # Load a modified client interface that looks like the interface used # from the minion, but uses remote execution # - self.functions = salt.client.FunctionWrapper(self.opts, self.minion) + self.functions = salt.client.FunctionWrapper( + self.opts, + self.opts['id'] + ) # Load the states, but they should not be used in this class apart # from inspection self.states = salt.loader.states(self.opts, self.functions) @@ -1233,13 +1238,13 @@ def __init__(self, opts, grains, id_, env=None): opts = copy.deepcopy(opts) opts['file_client'] = 'local' opts['file_roots'] = opts['master_roots'] - opts['id'] = minion + opts['id'] = id_ opts['grains'] = grains opts['environment'] = env self.client = salt.fileclient.get_file_client(opts) BaseHighState.__init__(self, opts) # Use the master state object - self.state = MasterState(self.opts, minion) + self.state = MasterState(self.opts, grains) self.matcher = salt.minion.Matcher(self.opts) From a6731a1d987ff89c753a6f80ad6cbbf72a90f03b Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Thu, 15 Mar 2012 23:24:37 -0600 Subject: [PATCH 384/598] retrive the actual list of functions --- salt/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/client.py b/salt/client.py index c1027928d6b6..01062f8eda35 100644 --- a/salt/client.py +++ b/salt/client.py @@ -724,7 +724,7 @@ def __load_functions(self): ''' Find out what functions are available on the minion ''' - return set(self.local.cmd(self.minion, 'sys.functions')) + return set(self.local.cmd(self.minion, 'sys.list_functions')) def run_key(self, key): ''' From 2cb75d0cd0c4d0ed8d95e474bdd89a1f2be98462 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Thu, 15 Mar 2012 23:26:49 -0600 Subject: [PATCH 385/598] Update salt/modules/pip.py --- salt/modules/pip.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index 41e1c7d84d0e..d0536df8a48f 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -17,7 +17,8 @@ def _get_pip_bin(pip, env): else: return pip if pip else __opts__['pip_bin'] -def install(packages, +def install(packages=None, + requirements=None, bin_env=None, log=None, proxy=None, @@ -66,15 +67,6 @@ def install(packages, salt '*' pip.install <package name> /path/to/pip_bin ''' - packages = packages.split(",") - - if len(packages) == 1 and packages[0].endswith('.txt'): - # we are installing a requirements file - pkg = "-r {reqs}".format(reqs=packages[0]) - else: - # we have a list of packages to install - pkg = ' '.join(packages) - if not bin_env: pip_bin = 'pip' else: @@ -83,10 +75,18 @@ def install(packages, pip_bin = os.path.join(bin_env, 'bin', 'pip') else: pip_bin = bin_env - - cmd = '{pip_bin} install {pkg}'.format( - pip_bin=pip_bin, - pkg=pkg) + + cmd = '{pip_bin} install'.format(pip_bin=pip_bin) + + if packages: + pkg = packages.replace(",", " ") + cmd = '{cmd} {pkg}'.format( + cmd=cmd, pkg=pkg) + + if requirements: + cmd = '{cmd} --requirements{requirements}'.format( + cmd=cmd, requirements=requirements} + if log: try: From 01691f890c3eb04d5fe8ca879e7aae5de335eb4c Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 15 Mar 2012 22:35:42 -0700 Subject: [PATCH 386/598] Add 'service.enable' and 'service.disable' for upstart get_enabled() and get_disabled() are really tricky as upstart / debian^Wubuntu have no way to do the equiv of: chkconfig --list --- salt/modules/upstart.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/salt/modules/upstart.py b/salt/modules/upstart.py index 6135050e7037..40a421e00bd9 100644 --- a/salt/modules/upstart.py +++ b/salt/modules/upstart.py @@ -7,6 +7,8 @@ ''' import os +import re +from salt.exceptions import CommandNotFoundError def __virtual__(): @@ -74,3 +76,36 @@ def status(name, sig=None): ''' cmd = 'service {0} status'.format(name) return not __salt__['cmd.retcode'](cmd) + + +def _get_service_exec(): + executable = 'update-rc.d' + if not __salt__['cmd.has_exec'](executable): + raise CommandNotFoundError('Missing {0}'.format(executable)) + return executable + + +def enable(name): + ''' + Enable the named service to start at boot + + CLI Example:: + + salt '*' service.enable <service name> + ''' + executable = _get_service_exec() + cmd = '{0} -n -f {1} defaults'.format(executable, name) + return not __salt__['cmd.retcode'](cmd) + + +def disable(name): + ''' + Disable the named service from starting on boot + + CLI Example:: + + salt '*' service.disable <service name> + ''' + executable = _get_service_exec() + cmd = '{0} -n -f {1} defaults'.format(executable, name) + return not __salt__['cmd.retcode'](cmd) From 49e698695759b2e0d3fc59536cb034cb84dab285 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Thu, 15 Mar 2012 23:40:46 -0600 Subject: [PATCH 387/598] Update salt/modules/pip.py --- salt/modules/pip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index d0536df8a48f..eeb709dac444 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -77,12 +77,12 @@ def install(packages=None, pip_bin = bin_env cmd = '{pip_bin} install'.format(pip_bin=pip_bin) - + print cmd if packages: pkg = packages.replace(",", " ") cmd = '{cmd} {pkg}'.format( cmd=cmd, pkg=pkg) - + print cmd if requirements: cmd = '{cmd} --requirements{requirements}'.format( cmd=cmd, requirements=requirements} From 4c66b236dfcce9807799c4332ef4082db875eae9 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Thu, 15 Mar 2012 23:44:14 -0600 Subject: [PATCH 388/598] Update salt/modules/pip.py --- salt/modules/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index eeb709dac444..9c2255584a2b 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -85,7 +85,7 @@ def install(packages=None, print cmd if requirements: cmd = '{cmd} --requirements{requirements}'.format( - cmd=cmd, requirements=requirements} + cmd=cmd, requirements=requirements) if log: From 7373a787a96c5c17e86fc793ca76b29cd71d60be Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Thu, 15 Mar 2012 23:58:10 -0600 Subject: [PATCH 389/598] Update salt/modules/pip.py --- salt/modules/pip.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index 9c2255584a2b..e168cf73f0e3 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -60,11 +60,13 @@ def install(packages=None, salt '*' pip.install <package name>,<package2 name> - salt '*' pip.install /path/to/requirements.txt + salt '*' pip.install requirements=/path/to/requirements.txt - salt '*' pip.install <package name> /path/to/virtualenv + salt '*' pip.install <package name> bin_env=/path/to/virtualenv - salt '*' pip.install <package name> /path/to/pip_bin + salt '*' pip.install <package name> bin_env=/path/to/pip_bin + + salt '*' pip.install markdown,django editable=git+https://github.com/worldcompany/djangoembed.git#egg=djangoembed upgrade=True no_deps=True ''' if not bin_env: @@ -74,7 +76,7 @@ def install(packages=None, if os.path.exists(os.path.join(bin_env, 'bin', 'pip')): pip_bin = os.path.join(bin_env, 'bin', 'pip') else: - pip_bin = bin_env + pip_bin = bin_env cmd = '{pip_bin} install'.format(pip_bin=pip_bin) print cmd From 6c765dd6b567e9ebf1a8314e4aa90ab532717c67 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 00:08:58 -0600 Subject: [PATCH 390/598] Update salt/modules/pip.py --- salt/modules/pip.py | 62 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index e168cf73f0e3..d71dcff7b15e 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -48,13 +48,70 @@ def install(packages=None, packages globally or to a virtualenv. packages - package(s) or requirements file + comma separated list of packages to install + requirements + path to requirements bin_env path to pip bin or path to virtualenv. If doing a system install, and want to use a specific pip bin (pip-2.7, pip-2.6, etc..) just specify the pip bin you want. If installing into a virtualenv, just use the path to the virtualenv (/home/code/path/to/virtualenv/) + log + Log file where a complete (maximum verbosity) record will be kept + proxy + Specify a proxy in the form + user:passwd@proxy.server:port. Note that the + user:password@ is optional and required only if you + are behind an authenticated proxy. If you provide + user@proxy.server:port then you will be prompted for a + password. + timeout + Set the socket timeout (default 15 seconds) + editable + install something editable(ie git+https://github.com/worldcompany/djangoembed.git#egg=djangoembed) + find_links + URL to look for packages at + index_url + Base URL of Python Package Index + extra_index_url + Extra URLs of package indexes to use in addition to ``index_url`` + no_index + Ignore package index + mirrors + Specific mirror URLs to query (automatically adds --use-mirrors) + build + Unpack packages into ``build`` dir + target + Install packages into ``target`` dir + download + Download packages into ``download`` instead of installing them + download_cache + Cache downloaded packages in ``download_cache`` dir + source + Check out ``editable`` packages into ``source`` dir + upgrade + Upgrade all packages to the newest available version + force_reinstall + When upgrading, reinstall all packages even if they are already up-to-date. + ignore_installed + Ignore the installed packages (reinstalling instead) + no_deps + Ignore package dependencies + no_install + Download and unpack all packages, but don't actually install them + no_download + Don't download any packages, just install the ones + already downloaded (completes an install run with + --no-install) + install_options + Extra arguments to be supplied to the setup.py install + command (use like --install-option="--install- + scripts=/usr/local/bin"). Use multiple --install- + option options to pass multiple options to setup.py + install. If you are using an option with a directory + path, be sure to use absolute path. + CLI Example:: @@ -65,8 +122,11 @@ def install(packages=None, salt '*' pip.install <package name> bin_env=/path/to/virtualenv salt '*' pip.install <package name> bin_env=/path/to/pip_bin + + Comlicated CLI example:: salt '*' pip.install markdown,django editable=git+https://github.com/worldcompany/djangoembed.git#egg=djangoembed upgrade=True no_deps=True + ''' if not bin_env: From 82aa81dbf6ec2808651458cd33939075f69dc19f Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Thu, 15 Mar 2012 23:12:20 -0700 Subject: [PATCH 391/598] Add 'Required python modules' docstring to ps --- salt/modules/ps.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/modules/ps.py b/salt/modules/ps.py index 11afcfae0ca9..1310a04aabf8 100644 --- a/salt/modules/ps.py +++ b/salt/modules/ps.py @@ -1,6 +1,8 @@ ''' A salt interface to psutil, a system and process library. See http://code.google.com/p/psutil. + +Required python modules: psutil ''' import sys From 56d325904e080fce6972fbfb04bdaca69aa20f55 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 00:13:09 -0600 Subject: [PATCH 392/598] Update salt/modules/pip.py --- salt/modules/pip.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index d71dcff7b15e..2e96adca19e5 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -1,21 +1,24 @@ ''' Install Python packages with pip to either the system or a virtualenv ''' -__opts__ = { - 'pip_bin': 'pip', -} import os -def _get_pip_bin(pip, env): +def _get_pip_bin(bin_env): ''' Return the pip command to call, either from a virtualenv, an argument passed in, or from the global modules options ''' - if env: - return os.path.join(env, 'bin', 'pip') + if not bin_env: + pip_bin = 'pip' else: - return pip if pip else __opts__['pip_bin'] + # try to get pip bin from env + if os.path.exists(os.path.join(bin_env, 'bin', 'pip')): + pip_bin = os.path.join(bin_env, 'bin', 'pip') + else: + pip_bin = bin_env + return pip_bin + def install(packages=None, requirements=None, @@ -128,15 +131,7 @@ def install(packages=None, salt '*' pip.install markdown,django editable=git+https://github.com/worldcompany/djangoembed.git#egg=djangoembed upgrade=True no_deps=True ''' - - if not bin_env: - pip_bin = 'pip' - else: - # try to get pip bin from env - if os.path.exists(os.path.join(bin_env, 'bin', 'pip')): - pip_bin = os.path.join(bin_env, 'bin', 'pip') - else: - pip_bin = bin_env + pip_bin = _get_pip_bin(bin_env) cmd = '{pip_bin} install'.format(pip_bin=pip_bin) print cmd @@ -248,7 +243,7 @@ def install(packages=None, return __salt__['cmd.run'](cmd) -def freeze(env='', pip_bin=''): +def freeze(bin_env=None): ''' Return a list of installed packages either globally or in the specified virtualenv @@ -262,6 +257,8 @@ def freeze(env='', pip_bin=''): to the pip that is installed in the virtualenv. This option can also be set in the minion config file as ``pip.pip_bin``. ''' - cmd = '{0} freeze'.format(_get_pip_bin(pip_bin, env)) + pip_bin = _get_pip_bin(bin_env) + + cmd = '{0} freeze'.format(pip_bin) return __salt__['cmd.run'](cmd).split('\n') From d68e381371c46857b6cc6fc77531a640a234f4d8 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 00:28:24 -0600 Subject: [PATCH 393/598] adding uninstall method --- salt/modules/pip.py | 150 +++++++++++++++++++++++++++++++++----------- 1 file changed, 112 insertions(+), 38 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index 2e96adca19e5..af70188c38a6 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -16,9 +16,9 @@ def _get_pip_bin(bin_env): if os.path.exists(os.path.join(bin_env, 'bin', 'pip')): pip_bin = os.path.join(bin_env, 'bin', 'pip') else: - pip_bin = bin_env + pip_bin = bin_env return pip_bin - + def install(packages=None, requirements=None, @@ -50,7 +50,7 @@ def install(packages=None, Install packages individually or from a pip requirements file. Install packages globally or to a virtualenv. - packages + packages comma separated list of packages to install requirements path to requirements @@ -114,7 +114,7 @@ def install(packages=None, option options to pass multiple options to setup.py install. If you are using an option with a directory path, be sure to use absolute path. - + CLI Example:: @@ -125,25 +125,23 @@ def install(packages=None, salt '*' pip.install <package name> bin_env=/path/to/virtualenv salt '*' pip.install <package name> bin_env=/path/to/pip_bin - + Comlicated CLI example:: - + salt '*' pip.install markdown,django editable=git+https://github.com/worldcompany/djangoembed.git#egg=djangoembed upgrade=True no_deps=True - + ''' - pip_bin = _get_pip_bin(bin_env) - - cmd = '{pip_bin} install'.format(pip_bin=pip_bin) - print cmd + + cmd = '{0} install'.format(_get_pip_bin(bin_env)) + if packages: pkg = packages.replace(",", " ") - cmd = '{cmd} {pkg}'.format( + cmd = '{cmd} {pkg} '.format( cmd=cmd, pkg=pkg) - print cmd + if requirements: - cmd = '{cmd} --requirements{requirements}'.format( + cmd = '{cmd} --requirements{requirements} '.format( cmd=cmd, requirements=requirements) - if log: try: @@ -151,11 +149,11 @@ def install(packages=None, os.path.exists(log) except IOError: raise IOError("'%s' is not writeable" % log) - cmd = '{cmd} --{log}'.format( + cmd = '{cmd} --{log} '.format( cmd=cmd, log=log) if proxy: - cmd = '{cmd} --proxy={proxy}'.format( + cmd = '{cmd} --proxy={proxy} '.format( cmd=cmd, proxy=proxy) if timeout: @@ -163,13 +161,13 @@ def install(packages=None, int(timeout) except ValueError: raise ValueError("'%s' is not a valid integer base 10.") - cmd = '{cmd} --timeout={timeout}'.format( + cmd = '{cmd} --timeout={timeout} '.format( cmd=cmd, timeout=timeout) if editable: if editable.find('egg') == -1: raise Exception('You must specify an egg for this editable') - cmd = '{cmd} --editable={editable}'.format( + cmd = '{cmd} --editable={editable} '.format( cmd=cmd, editable=editable) if find_links: @@ -181,68 +179,145 @@ def install(packages=None, if index_url: if not index_url.startswith("http://"): raise Exception("'%s' must be a valid url" % index_url) - cmd = '{cmd} --index_url={index_url}'.format( + cmd = '{cmd} --index_url={index_url} '.format( cmd=cmd, index_url=index_url) if extra_index_url: if not extra_index_url.startswith("http://"): raise Exception("'%s' must be a valid url" % extra_index_url) - cmd = '{cmd} --extra_index_url={extra_index_url}'.format( + cmd = '{cmd} --extra_index_url={extra_index_url} '.format( cmd=cmd, extra_index_url=extra_index_url) if no_index: - cmd = '{cmd} --no-index'.format(cmd=cmd) + cmd = '{cmd} --no-index '.format(cmd=cmd) if mirrors: if not mirrors.startswith("http://"): raise Exception("'%s' must be a valid url" % mirrors) - cmd = '{cmd} --use-mirrors --mirrors={mirrors}'.format( + cmd = '{cmd} --use-mirrors --mirrors={mirrors} '.format( cmd=cmd, mirrors=mirrors) if build: - cmd = '{cmd} --build={build}'.format( + cmd = '{cmd} --build={build} '.format( cmd=cmd, build=build) if target: - cmd = '{cmd} --target={target}'.format( + cmd = '{cmd} --target={target} '.format( cmd=cmd, target=target) if download: - cmd = '{cmd} --download={download}'.format( + cmd = '{cmd} --download={download} '.format( cmd=cmd, download=download) if download_cache: - cmd = '{cmd} --download_cache={download_cache}'.format( + cmd = '{cmd} --download_cache={download_cache} '.format( cmd=cmd, download_cache=download_cache) if source: - cmd = '{cmd} --source={source}'.format( + cmd = '{cmd} --source={source} '.format( cmd=cmd, source=source) if upgrade: - cmd = '{cmd} --upgrade'.format(cmd=cmd) + cmd = '{cmd} --upgrade '.format(cmd=cmd) if force_reinstall: - cmd = '{cmd} --force-reinstall'.format(cmd=cmd) + cmd = '{cmd} --force-reinstall '.format(cmd=cmd) if ignore_installed: - cmd = '{cmd} --ignore-installed'.format(cmd=cmd) + cmd = '{cmd} --ignore-installed '.format(cmd=cmd) if no_deps: - cmd = '{cmd} --no-deps'.format(cmd=cmd) + cmd = '{cmd} --no-deps '.format(cmd=cmd) if no_install: - cmd = '{cmd} --no-install'.format(cmd=cmd) + cmd = '{cmd} --no-install '.format(cmd=cmd) if no_download: - cmd = '{cmd} --no-download'.format(cmd=cmd) + cmd = '{cmd} --no-download '.format(cmd=cmd) if install_options: - cmd = '{cmd} --install-options={install_options}'.format( + cmd = '{cmd} --install-options={install_options} '.format( cmd=cmd, install_options=install_options) return __salt__['cmd.run'](cmd) + +def uninstall(packages=None, + requirements=None, + bin_env=None, + log=None, + proxy=None, + timeout=None): + ''' + Uninstall packages with pip + + Uninstall packages individually or from a pip requirements file. Uninstall + packages globally or from a virtualenv. + + packages + comma separated list of packages to install + requirements + path to requirements + bin_env + path to pip bin or path to virtualenv. If doing an uninstall from + the system python and want to use a specific pip bin (pip-2.7, + pip-2.6, etc..) just specify the pip bin you want. + If uninstalling from a virtualenv, just use the path to the virtualenv + (/home/code/path/to/virtualenv/) + log + Log file where a complete (maximum verbosity) record will be kept + proxy + Specify a proxy in the form + user:passwd@proxy.server:port. Note that the + user:password@ is optional and required only if you + are behind an authenticated proxy. If you provide + user@proxy.server:port then you will be prompted for a + password. + timeout + Set the socket timeout (default 15 seconds) + + + CLI Example:: + + salt '*' pip.uninstall <package name>,<package2 name> + + salt '*' pip.uninstall requirements=/path/to/requirements.txt + + salt '*' pip.uninstall <package name> bin_env=/path/to/virtualenv + + salt '*' pip.uninstall <package name> bin_env=/path/to/pip_bin + + ''' + cmd = '{0} uninstall -y '.format(_get_pip_bin(bin_env)) + + if requirements: + cmd = '{cmd} --requirements{requirements} '.format( + cmd=cmd, requirements=requirements) + + if log: + try: + # TODO make this check if writeable + os.path.exists(log) + except IOError: + raise IOError("'%s' is not writeable" % log) + cmd = '{cmd} --{log} '.format( + cmd=cmd, log=log) + + if proxy: + cmd = '{cmd} --proxy={proxy} '.format( + cmd=cmd, proxy=proxy) + + if timeout: + try: + int(timeout) + except ValueError: + raise ValueError("'%s' is not a valid integer base 10.") + cmd = '{cmd} --timeout={timeout} '.format( + cmd=cmd, timeout=timeout) + + return __salt__['cmd.run'](cmd).split('\n') + + def freeze(bin_env=None): ''' Return a list of installed packages either globally or in the specified @@ -257,8 +332,7 @@ def freeze(bin_env=None): to the pip that is installed in the virtualenv. This option can also be set in the minion config file as ``pip.pip_bin``. ''' - pip_bin = _get_pip_bin(bin_env) - - cmd = '{0} freeze'.format(pip_bin) + + cmd = '{0} freeze'.format(_get_pip_bin(bin_env)) return __salt__['cmd.run'](cmd).split('\n') From fb1697b80d6de6df41668c6c2bc52846095e3eef Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 16 Mar 2012 00:29:02 -0600 Subject: [PATCH 394/598] transfer more data for master state compile --- salt/master.py | 6 +++--- salt/state.py | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/salt/master.py b/salt/master.py index 2f8e8fd4a547..b30a6bf08d99 100644 --- a/salt/master.py +++ b/salt/master.py @@ -558,13 +558,13 @@ def _master_state(self, load): ''' Call the master to compile a master side highstate ''' - if 'id' not in load or 'grains' not in load or 'env' not in load: + if 'opts' not in load or 'grains' not in load: return False return salt.state.master_compile( self.opts, load['grains'], - load['id'], - load['env']) + load['opts']['id'], + load['opts']['environment']) def _return(self, load): ''' diff --git a/salt/state.py b/salt/state.py index ee4b20695060..3c85fffd6c42 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1255,7 +1255,6 @@ class RemoteHighState(object): def __init__(self, opts, grains): self.opts = opts self.grains = grains - self.id_ = opts['id'] self.serial = salt.payload.Serial(self.opts) self.auth = salt.crypt.SAuth(opts) self.socket = self.__get_socket() @@ -1274,9 +1273,8 @@ def compile_master(self): Return the state data from the master ''' payload = {'enc': 'aes'} - load = {'id': self.id_, - 'grains': self.grains, - 'env': self.opts['environment'], + load = {'grains': self.grains, + 'opts': self.opts, 'cmd': '_master_state'} payload['load'] = self.auth.crypticle.dumps(load) self.socket.send(self.serial.dumps(payload)) From dc124d5492f06a974988f26e394c982471c86d27 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 00:29:39 -0600 Subject: [PATCH 395/598] Update salt/modules/pip.py --- salt/modules/pip.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index af70188c38a6..8df90c6ed161 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -323,14 +323,12 @@ def freeze(bin_env=None): Return a list of installed packages either globally or in the specified virtualenv - env : None - The path to a virtualenv that pip should install to. This option takes - precendence over the ``pip_bin`` argument. - pip_bin : 'pip' - The name (and optionally path) of the pip command to call. This option - will be ignored if the ``env`` argument is given since it will default - to the pip that is installed in the virtualenv. This option can also be - set in the minion config file as ``pip.pip_bin``. + bin_env + path to pip bin or path to virtualenv. If doing an uninstall from + the system python and want to use a specific pip bin (pip-2.7, + pip-2.6, etc..) just specify the pip bin you want. + If uninstalling from a virtualenv, just use the path to the virtualenv + (/home/code/path/to/virtualenv/) ''' cmd = '{0} freeze'.format(_get_pip_bin(bin_env)) From c17722dac41a4c589c450aca2e9213936e5c5a6e Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 00:32:27 -0600 Subject: [PATCH 396/598] adding processor for packages arg --- salt/modules/pip.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index 8df90c6ed161..51679119dbdb 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -289,7 +289,12 @@ def uninstall(packages=None, ''' cmd = '{0} uninstall -y '.format(_get_pip_bin(bin_env)) - + + if packages: + pkg = packages.replace(",", " ") + cmd = '{cmd} {pkg} '.format( + cmd=cmd, pkg=pkg) + if requirements: cmd = '{cmd} --requirements{requirements} '.format( cmd=cmd, requirements=requirements) From 29d0f1cff2eb15fd2eada596a8a7b1ea2d9d6b6c Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 00:39:10 -0600 Subject: [PATCH 397/598] adding list skelton function --- salt/modules/pip.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index 51679119dbdb..cf0daefbf534 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -339,3 +339,11 @@ def freeze(bin_env=None): cmd = '{0} freeze'.format(_get_pip_bin(bin_env)) return __salt__['cmd.run'](cmd).split('\n') + + +def list(name, bin_env=None): + ''' + Filter list of instaslled apps from ``freeze`` and check to see if ``name`` + exists in the list of packages installed. + ''' + pass From ea34586eab718182430994f04de5f39aa94203ab Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 06:45:39 +0000 Subject: [PATCH 398/598] adding some states for pip... not sure if it works --- salt/states/pip.py | 73 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 salt/states/pip.py diff --git a/salt/states/pip.py b/salt/states/pip.py new file mode 100644 index 000000000000..91ecf08450f0 --- /dev/null +++ b/salt/states/pip.py @@ -0,0 +1,73 @@ +''' +Management of python packages +============================= + +A state module to manage system installed python packages + +.. code-block:: yam + + virtualenvwrapper: + pip: + - installed + - version: 3.0.1 +''' + + +def installed(name, pip_bin=None): + ''' + Make sure the package is installed + + name + The name of the python package to install + pip_bin : None + the pip executable to use + + ''' + if not pip_bin: + pip_bin='pip' + + ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} + if name in __salt__['pip.list'](name, pip_bin): + ret['result'] = True + ret['comment'] = 'Package already installed' + return ret + + if __salt__['pip.install'](packages=name, bin_env=pip_bin): + ret['result'] = True + ret['changes'][name] = 'Installed' + ret['comment'] = 'Package was successfully installed' + else: + ret['result'] = False + ret['comment'] = 'Could not install package' + + return ret + + +def removed(name, pip_bin=None): + """ + Make sure that a package is not installed. + + name + The name of the package to uninstall + For RVM installations: the ruby version and gemset to target. + pip_bin : None + the pip executable to use + """ + if not pip_bin: + pip_bin='pip' + + ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} + if name not in __salt__["pip.list"](packages=name, bin_env=bin_pip): + ret["result"] = True + ret["comment"] = "Pacakge is not installed." + return ret + + if __salt__["pip.uninstall"](packages=name, bin_env=pip_bin): + ret["result"] = True + ret["changes"][name] = "Removed" + ret["comment"] = "Package was successfully removed." + else: + ret["result"] = False + ret["comment"] = "Could not remove package." + return ret + From 9ac93f1e9c5df7d6048f11d3b0a7fcd1490dad2a Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 06:45:52 +0000 Subject: [PATCH 399/598] change no_site_packages to default to True --- salt/states/virtualenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/virtualenv.py b/salt/states/virtualenv.py index da5edc2d2cad..f5ede2961098 100644 --- a/salt/states/virtualenv.py +++ b/salt/states/virtualenv.py @@ -10,7 +10,7 @@ def manage(name, venv_bin='virtualenv', requirements='', - no_site_packages=False, + no_site_packages=True, system_site_packages=False, clear=False, python='', From c1f4842b3497c5f565f435ad766635f6926073fa Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 16 Mar 2012 00:57:58 -0600 Subject: [PATCH 400/598] normalize configuration data passed to MasterHighState --- salt/state.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/salt/state.py b/salt/state.py index 3c85fffd6c42..a71b5c7dcaf4 100644 --- a/salt/state.py +++ b/salt/state.py @@ -127,11 +127,11 @@ def format_log(ret): log.info(str(ret)) -def master_compile(opts, grains, id_, env): +def master_compile(master_opts, minion_opts, grains, id_, env): ''' Compile the master side low state data, and build the hidden state file ''' - st_ = MasterHighState(opts, grains, id_, env) + st_ = MasterHighState(master_opts, minion_opts, grains, id_, env) return st_.compile_highstate() def ishashable(obj): @@ -1233,14 +1233,15 @@ class MasterHighState(BaseHighState): ''' Execute highstate compilation from the master ''' - def __init__(self, opts, grains, id_, env=None): + def __init__(self, master_opts, minion_opts, grains, id_, env=None): # Force the fileclient to be local - opts = copy.deepcopy(opts) + opts = copy.deepcopy(minion_opts) opts['file_client'] = 'local' - opts['file_roots'] = opts['master_roots'] + opts['file_roots'] = master_opts['master_roots'] + opts['renderer'] = master_opts['renderer'] + opts['state_top'] = master_opts['state_top'] opts['id'] = id_ opts['grains'] = grains - opts['environment'] = env self.client = salt.fileclient.get_file_client(opts) BaseHighState.__init__(self, opts) # Use the master state object From ee8b523121b76225e0e2f22b4f85cfa26f1ce433 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 16 Mar 2012 01:11:36 -0600 Subject: [PATCH 401/598] Repair missing argument in passing minion data to states --- salt/master.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/master.py b/salt/master.py index b30a6bf08d99..c48fd28711c8 100644 --- a/salt/master.py +++ b/salt/master.py @@ -562,6 +562,7 @@ def _master_state(self, load): return False return salt.state.master_compile( self.opts, + load['opts'], load['grains'], load['opts']['id'], load['opts']['environment']) From 0ed1253036ffacc59ba89648458208cda9798ab4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 16 Mar 2012 01:21:39 -0600 Subject: [PATCH 402/598] get the actual list, not just the top of the dict --- salt/client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/client.py b/salt/client.py index 01062f8eda35..ec72eef5a3e8 100644 --- a/salt/client.py +++ b/salt/client.py @@ -724,7 +724,12 @@ def __load_functions(self): ''' Find out what functions are available on the minion ''' - return set(self.local.cmd(self.minion, 'sys.list_functions')) + return set( + self.local.cmd( + self.minion, + 'sys.list_functions' + ).get(self.minion, []) + ) def run_key(self, key): ''' From 5f4d7909d45739198ff2e9b9a741dbaf723a5519 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 07:44:17 +0000 Subject: [PATCH 403/598] adding list method for use in pip state --- salt/modules/pip.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index cf0daefbf534..cdd07aa2e41f 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -341,9 +341,23 @@ def freeze(bin_env=None): return __salt__['cmd.run'](cmd).split('\n') -def list(name, bin_env=None): +def list(prefix='', bin_env=None): ''' - Filter list of instaslled apps from ``freeze`` and check to see if ``name`` + Filter list of instaslled apps from ``freeze`` and check to see if ``prefix`` exists in the list of packages installed. ''' - pass + packages = {} + cmd = '{0} freeze'.format(_get_pip_bin(bin_env)) + for line in __salt__['cmd.run'](cmd).split("\n"): + if len(line.split("==")) >= 2: + name = line.split("==")[0] + version = line.split("==")[1] + if prefix: + if line.lower().startswith(prefix.lower()): + packages[name]=version + else: + packages[name]=version + return packages + + + From afd040353f9e1db02499c07ac434f1d996838ecd Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 01:47:49 -0600 Subject: [PATCH 404/598] Update salt/states/pip.py --- salt/states/pip.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/states/pip.py b/salt/states/pip.py index 91ecf08450f0..0c380e725d4b 100644 --- a/salt/states/pip.py +++ b/salt/states/pip.py @@ -4,7 +4,7 @@ A state module to manage system installed python packages -.. code-block:: yam +.. code-block:: yaml virtualenvwrapper: pip: @@ -49,7 +49,6 @@ def removed(name, pip_bin=None): name The name of the package to uninstall - For RVM installations: the ruby version and gemset to target. pip_bin : None the pip executable to use """ From f8edc920bcead9bfdbb620415653b6dd72ab9e36 Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Fri, 16 Mar 2012 09:49:13 -0600 Subject: [PATCH 405/598] Check for existence of 'virtual' in __grains__ first. This was breaking the minion on Windows. --- salt/modules/kvm_hyper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/modules/kvm_hyper.py b/salt/modules/kvm_hyper.py index 793844a51e07..e1e34c218c1a 100644 --- a/salt/modules/kvm_hyper.py +++ b/salt/modules/kvm_hyper.py @@ -42,6 +42,8 @@ def __virtual__(): ''' Apply this module as the hyper module if the minion is a kvm hypervisor ''' + if 'virtual' not in __grains__: + return False if __grains__['virtual'] != 'physical': return False if 'kvm_' not in open('/proc/modules').read(): From 5553c930cf0113281af5968d2fdad49b109e64f7 Mon Sep 17 00:00:00 2001 From: Nate Smith <nathaniel.smith@thescore.com> Date: Fri, 16 Mar 2012 15:56:09 -0400 Subject: [PATCH 406/598] Fix an issue with directory creation --- salt/fileclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index f44965333e77..6ed54ae9dea2 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -255,7 +255,7 @@ def get_url(self, url, dest, makedirs=False, env='base'): env, os.path.join( url_data.netloc, - os.path.relpath(url_data.path, '/')) + os.path.relpath(os.path.relpath(url_data.path, '/'), '..') ) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): From f1862348cdc00dbf34f6f9d2a9e35c9981801ee0 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 16 Mar 2012 16:14:52 -0600 Subject: [PATCH 407/598] Fix spelling error --- salt/modules/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/apt.py b/salt/modules/apt.py index 2bd39b9cbfa8..f33c5fe24b18 100644 --- a/salt/modules/apt.py +++ b/salt/modules/apt.py @@ -22,7 +22,7 @@ def __init__(): env_vars = { 'APT_LISTBUGS_FRONTEND': 'none', 'APT_LISTCHANGES_FRONTEND': 'none', - 'DEBIAN_FRONTENT': 'noninteractive', + 'DEBIAN_FRONTEND': 'noninteractive', } # Export these puppies so they persist os.environ.update(env_vars) From 69efe9173b3bf0f4a56a3315beaf2f30c7b5efdb Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Fri, 16 Mar 2012 16:52:29 -0600 Subject: [PATCH 408/598] missing paren --- salt/fileclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index 6ed54ae9dea2..9c30b7b4cf8c 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -256,7 +256,7 @@ def get_url(self, url, dest, makedirs=False, env='base'): os.path.join( url_data.netloc, os.path.relpath(os.path.relpath(url_data.path, '/'), '..') - ) + )) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) From f01245b8e73d1f82431ddfef5d969c4537bfa839 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 17:24:13 -0600 Subject: [PATCH 409/598] renaming packages to pkgs, adding env var that will help handle backwards compatibility issues, updating docs to support new parameter names --- salt/modules/pip.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index cdd07aa2e41f..22248c6a8f47 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -20,8 +20,9 @@ def _get_pip_bin(bin_env): return pip_bin -def install(packages=None, +def install(pkgs=None, requirements=None, + env=None, bin_env=None, log=None, proxy=None, @@ -50,7 +51,7 @@ def install(packages=None, Install packages individually or from a pip requirements file. Install packages globally or to a virtualenv. - packages + pkgs comma separated list of packages to install requirements path to requirements @@ -60,6 +61,8 @@ def install(packages=None, specify the pip bin you want. If installing into a virtualenv, just use the path to the virtualenv (/home/code/path/to/virtualenv/) + env + depreicated, use bin_env now log Log file where a complete (maximum verbosity) record will be kept proxy @@ -131,11 +134,19 @@ def install(packages=None, salt '*' pip.install markdown,django editable=git+https://github.com/worldcompany/djangoembed.git#egg=djangoembed upgrade=True no_deps=True ''' + # Switching from using `pip_bin` and `env` to just `bin_env` + # cause using an env and a pip bin that's not in the env could + # be problematic. + # Still using the `env` variable, for backwards compatiblity sake + # but going fwd you should specify either a pip bin or an env with + # the `bin_env` argument and we'll take care of the rest. + if env and not bin_env: + bin_env = env cmd = '{0} install'.format(_get_pip_bin(bin_env)) - if packages: - pkg = packages.replace(",", " ") + if pkgs: + pkg = pkgs.replace(",", " ") cmd = '{cmd} {pkg} '.format( cmd=cmd, pkg=pkg) @@ -242,7 +253,7 @@ def install(packages=None, return __salt__['cmd.run'](cmd) -def uninstall(packages=None, +def uninstall(pkgs=None, requirements=None, bin_env=None, log=None, @@ -254,7 +265,7 @@ def uninstall(packages=None, Uninstall packages individually or from a pip requirements file. Uninstall packages globally or from a virtualenv. - packages + pkgs comma separated list of packages to install requirements path to requirements @@ -289,12 +300,12 @@ def uninstall(packages=None, ''' cmd = '{0} uninstall -y '.format(_get_pip_bin(bin_env)) - - if packages: - pkg = packages.replace(",", " ") + + if pkgs: + pkg = pkgs.replace(",", " ") cmd = '{cmd} {pkg} '.format( cmd=cmd, pkg=pkg) - + if requirements: cmd = '{cmd} --requirements{requirements} '.format( cmd=cmd, requirements=requirements) @@ -339,7 +350,7 @@ def freeze(bin_env=None): cmd = '{0} freeze'.format(_get_pip_bin(bin_env)) return __salt__['cmd.run'](cmd).split('\n') - + def list(prefix='', bin_env=None): ''' From 8d725ab50817808611de829b57f1ff9286dad828 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 17:30:49 -0600 Subject: [PATCH 410/598] making changes to the pip state so that some of the backwards compatible issues in the pip module are reflected here. --- salt/states/pip.py | 83 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/salt/states/pip.py b/salt/states/pip.py index 0c380e725d4b..4193b62c32fd 100644 --- a/salt/states/pip.py +++ b/salt/states/pip.py @@ -13,18 +13,45 @@ ''' -def installed(name, pip_bin=None): +def installed(name, + pip_bin=None, + requirements=None, + env=None, + bin_env=None, + log=None, + proxy=None, + timeout=None, + editable=None, + find_links=None, + index_url=None, + extra_index_url=None, + no_index=False, + mirrors=None, + build=None, + target=None, + download=None, + download_cache=None, + source=None, + upgrade=False, + force_reinstall=False, + ignore_installed=False, + no_deps=False, + no_install=False, + no_download=False, + install_options=None): ''' Make sure the package is installed name The name of the python package to install pip_bin : None - the pip executable to use + Deprecated, use bin_env + bin_env : None + the pip executable or virtualenv to use ''' - if not pip_bin: - pip_bin='pip' + if pip_bin and not bin_env: + bin_env = pip_bin ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} if name in __salt__['pip.list'](name, pip_bin): @@ -32,7 +59,30 @@ def installed(name, pip_bin=None): ret['comment'] = 'Package already installed' return ret - if __salt__['pip.install'](packages=name, bin_env=pip_bin): + if __salt__['pip.install'](pkgs=name, + requirements=requirements, + bin_env=bin_env, + log=log, + proxy=proxy, + timeout=timeout, + editable=editable, + find_links=find_links, + index_url=index_url, + extra_index_url=extra_index_url, + no_index=no_index, + mirrors=mirrors, + build=build, + target=target, + download=download, + download_cache=download_cache, + source=source, + upgrade=upgrade, + force_reinstall=force_reinstall, + ignore_installed=ignore_installed, + no_deps=no_deps, + no_install=no_install, + no_download=no_download, + install_options=install_options): ret['result'] = True ret['changes'][name] = 'Installed' ret['comment'] = 'Package was successfully installed' @@ -43,25 +93,34 @@ def installed(name, pip_bin=None): return ret -def removed(name, pip_bin=None): +def removed(name, + packages=None, + requirements=None, + bin_env=None, + log=None, + proxy=None, + timeout=None): """ Make sure that a package is not installed. name The name of the package to uninstall - pip_bin : None - the pip executable to use + bin_env : None + the pip executable or virtualenenv to use """ - if not pip_bin: - pip_bin='pip' ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} - if name not in __salt__["pip.list"](packages=name, bin_env=bin_pip): + if name not in __salt__["pip.list"](packages=name, bin_env=bin_env): ret["result"] = True ret["comment"] = "Pacakge is not installed." return ret - if __salt__["pip.uninstall"](packages=name, bin_env=pip_bin): + if __salt__["pip.uninstall"](packages=name, + requirements=requirements, + bin_env=bin_env, + log=log, + proxy=proxy, + timeout=timeout): ret["result"] = True ret["changes"][name] = "Removed" ret["comment"] = "Package was successfully removed." From fef4fce46b71f716239f637d41e37f99868dfecd Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 17:39:39 -0600 Subject: [PATCH 411/598] adding another deprecated comment, and adding a backwards compatability check on env as well --- salt/states/pip.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/states/pip.py b/salt/states/pip.py index 4193b62c32fd..fba036bfc085 100644 --- a/salt/states/pip.py +++ b/salt/states/pip.py @@ -46,12 +46,16 @@ def installed(name, The name of the python package to install pip_bin : None Deprecated, use bin_env + env : None + Deprecated, use bin_env bin_env : None the pip executable or virtualenv to use ''' if pip_bin and not bin_env: bin_env = pip_bin + elif env and not bin_env: + bin_env = env ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} if name in __salt__['pip.list'](name, pip_bin): From 539333b2c0dc5f2ff553f60ca85b7b0773f8c3c3 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 17:40:36 -0600 Subject: [PATCH 412/598] updating the virtualenv state to reflect teh changes to the api for pip module --- salt/states/virtualenv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/states/virtualenv.py b/salt/states/virtualenv.py index f5ede2961098..8531200a7176 100644 --- a/salt/states/virtualenv.py +++ b/salt/states/virtualenv.py @@ -62,7 +62,7 @@ def manage(name, # If it already exists, grab the version for posterity if venv_exists and clear: - ret['changes']['cleared_packages'] = __salt__['pip.freeze'](env=name) + ret['changes']['cleared_packages'] = __salt__['pip.freeze'](bin_env=name) ret['changes']['old'] = __salt__['cmd.run_stderr']( '{0} -V'.format(venv_py)).strip('\n') @@ -109,9 +109,9 @@ def manage(name, else: new_reqs = __salt__['cp.cache_local_file'](requirements) - before = set(__salt__['pip.freeze'](env=name)) - __salt__['pip.install'](requirements=new_reqs, env=name) - after = set(__salt__['pip.freeze'](env=name)) + before = set(__salt__['pip.freeze'](bin_env=name)) + __salt__['pip.install'](requirements=new_reqs, bin_env=name) + after = set(__salt__['pip.freeze'](bin_env=name)) new = list(after - before) old = list(before - after) From 771f2868f721f6bbea362271398cddc961b96705 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick.lang@gmail.com> Date: Fri, 16 Mar 2012 17:42:09 -0600 Subject: [PATCH 413/598] updatin doc string to reflect the method sig --- salt/modules/virtualenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/virtualenv.py b/salt/modules/virtualenv.py index f6678e8ff263..690f1f752106 100644 --- a/salt/modules/virtualenv.py +++ b/salt/modules/virtualenv.py @@ -27,7 +27,7 @@ def create(path, venv_bin : 'virtualenv' The name (and optionally path) of the virtualenv command. This can also be set globally in the minion config file as ``virtualenv.venv_bin``. - no_site_packages : False + no_site_packages : True Passthrough argument given to virtualenv system_site_packages : False Passthrough argument given to virtualenv From 2926779b371e3bbc3b35c16bcd4e300090886c51 Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Fri, 16 Mar 2012 22:46:39 -0600 Subject: [PATCH 414/598] Fix logging log was called without importing logging and setting up log --- salt/modules/reg.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/modules/reg.py b/salt/modules/reg.py index c64937788134..41700175da4d 100644 --- a/salt/modules/reg.py +++ b/salt/modules/reg.py @@ -14,8 +14,11 @@ has_windows_modules = False import salt.utils +import logging from salt.exceptions import CommandExecutionError +log = logging.getLogger(__name__) + class Registry(object): ''' Delay '_winreg' usage until this module is used From 4003bc752dc5a6d32eb2b7fb541b58de7a8b29a0 Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Fri, 16 Mar 2012 14:17:01 -0600 Subject: [PATCH 415/598] Add network runner. First function is wol (Wake On Lan) --- salt/runners/network.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 salt/runners/network.py diff --git a/salt/runners/network.py b/salt/runners/network.py new file mode 100644 index 000000000000..8b21c9cb7d71 --- /dev/null +++ b/salt/runners/network.py @@ -0,0 +1,35 @@ +''' +Network tools to run from the Master +''' + +import socket +import struct + + +def wol(mac, bcast='255.255.255.255', destport=9): + ''' + Send a "Magic Packet" to wake up a Minion + + CLI Example:: + + salt-run 08-00-27-13-69-77 + salt-run 080027136977 255.255.255.255 7 + salt-run 08:00:27:13:69:77 255.255.255.255 7 + ''' + if len(mac) == 12: + pass + elif len(mac) == 17: + sep = mac[2] + mac = mac.replace(sep, '') + else: + raise ValueError('Invalid MAC address') + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + dest = ('\\x' + mac[0:2]).decode('string_escape') + \ + ('\\x' + mac[2:4]).decode('string_escape') + \ + ('\\x' + mac[4:6]).decode('string_escape') + \ + ('\\x' + mac[6:8]).decode('string_escape') + \ + ('\\x' + mac[8:10]).decode('string_escape') + \ + ('\\x' + mac[10:12]).decode('string_escape') + s.sendto('\xff'*6 + dest*16, (bcast, destport)) + print "Sent packet" From 304957d0c9b40e66a3fabc896d3c972846cf3d98 Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Fri, 16 Mar 2012 23:37:02 -0600 Subject: [PATCH 416/598] Add Wake On Lan to master Allow the Master to send a "Magic Packet" to wake a minion up. --- salt/runners/network.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/runners/network.py b/salt/runners/network.py index 8b21c9cb7d71..10f62b49bbe4 100644 --- a/salt/runners/network.py +++ b/salt/runners/network.py @@ -3,8 +3,6 @@ ''' import socket -import struct - def wol(mac, bcast='255.255.255.255', destport=9): ''' @@ -30,6 +28,6 @@ def wol(mac, bcast='255.255.255.255', destport=9): ('\\x' + mac[4:6]).decode('string_escape') + \ ('\\x' + mac[6:8]).decode('string_escape') + \ ('\\x' + mac[8:10]).decode('string_escape') + \ - ('\\x' + mac[10:12]).decode('string_escape') - s.sendto('\xff'*6 + dest*16, (bcast, destport)) - print "Sent packet" + ('\\x' + mac[10:12]).decode('string_escape') + s.sendto('\xff'*6 + dest*16, (bcast, int(destport))) + print "Sent magic packet to minion." From 7dcf24f411720201481ff1b79badd82905b81d37 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 17 Mar 2012 01:10:46 -0600 Subject: [PATCH 417/598] Start release notes for 0.9.8 --- doc/topics/releases/0.9.8.rst | 85 +++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 doc/topics/releases/0.9.8.rst diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst new file mode 100644 index 000000000000..92f95fff9bde --- /dev/null +++ b/doc/topics/releases/0.9.8.rst @@ -0,0 +1,85 @@ +======================== +Salt 0.9.8 Release Notes +======================== + +Salt 0.9.8 is a big step forward, with many additions and enhancements, as +well as a number of precursours to advanced future developments. + +This version of Salt adds much more power to the command line, making the +old hard timeout isses a thing of the past, and adding kwargs support to +salt calls. These additions are also available in the salt client api, making +the available api tools much more powerful. + +The new pillar system allows for data to be stored on the master and +assigned to minions in a granular way similar to the state system. + +Additions to requisites making them much more powerful have been added and +improved errorchecking for sls files in the state system. A new provider +system has been added to allow for redirecting what modules function in +the background for individual states. + +Support for OpenSUSE has been added and support for Solaris has begun +serious development. + +The matcher and target systems have recieved a great deal of attention, the +default behavior of grain matching has changed slightly to reflect the regular +salt default behavior and the compound matcher system has been greatly refined. + +A number of impressive features with keyword arguments have been added to both +the cli and to the state system, making states much more powerful and flexible, +but available in a clean and simple way. + +The new batch size capability allows for executions to be rolled through the +targeted minions. + + +Major Features +============== + +Pillar +------ + +Pillar offers an interface to declare variable data on the master that is then +assigned to the minions. The pillar data is made available to all modules, +states, sls files etc. It is compiled on the master and is declared using the +existing render system. This means that learning pillar should be fairly +trivial to those already familiar with salt states. + +CLI Additions +------------- + +Better State Return Data +------------------------ + +Kwargs and States +----------------- + +Kwargs and the CLI +------------------ + +Matcher Refinements and Changes +------------------------------- + +Providers +--------- + +Requisite Glob Matching +----------------------- + +Batch Size +---------- + +In Progress Development +======================= + +Master Side State Compiling +--------------------------- + +While we feel strongly that the advantages gained with minion side state +compiling is very critical, it does prevent certian features what may be +desired. 0.9.8 has support for initial master side state compiling, but many +more components still need to be developed, it is hoped that these can be +finished for 0.9.9. + +The goal is that states can be compiled on both the master and the minion +allowing for compilation to be split between master and minion. From 65778e15b92df29a40146905f0b37fd79ac99cc0 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 08:39:51 -0700 Subject: [PATCH 418/598] Fix a few tyops --- doc/topics/releases/0.9.8.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 92f95fff9bde..b45669b2d4c8 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -3,10 +3,10 @@ Salt 0.9.8 Release Notes ======================== Salt 0.9.8 is a big step forward, with many additions and enhancements, as -well as a number of precursours to advanced future developments. +well as a number of precursors to advanced future developments. This version of Salt adds much more power to the command line, making the -old hard timeout isses a thing of the past, and adding kwargs support to +old hard timeout issues a thing of the past, and adding kwargs support to salt calls. These additions are also available in the salt client api, making the available api tools much more powerful. @@ -14,14 +14,14 @@ The new pillar system allows for data to be stored on the master and assigned to minions in a granular way similar to the state system. Additions to requisites making them much more powerful have been added and -improved errorchecking for sls files in the state system. A new provider +improved error checking for sls files in the state system. A new provider system has been added to allow for redirecting what modules function in the background for individual states. -Support for OpenSUSE has been added and support for Solaris has begun +Support for OpenSUSE has been added and support for Solaris has begun serious development. -The matcher and target systems have recieved a great deal of attention, the +The matcher and target systems have received a great deal of attention, the default behavior of grain matching has changed slightly to reflect the regular salt default behavior and the compound matcher system has been greatly refined. @@ -76,7 +76,7 @@ Master Side State Compiling --------------------------- While we feel strongly that the advantages gained with minion side state -compiling is very critical, it does prevent certian features what may be +compiling is very critical, it does prevent certain features what may be desired. 0.9.8 has support for initial master side state compiling, but many more components still need to be developed, it is hoped that these can be finished for 0.9.9. From 039b4f12f6113eef5fc8db6e74b62a2b3ed9e203 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 17 Mar 2012 09:48:35 -0600 Subject: [PATCH 419/598] Add catch for removal of module_refresh file --- salt/minion.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index 2c9069167869..7b9eae98c22d 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -420,7 +420,10 @@ def passive_refresh(self): self.opts['id'], self.opts['environment'], ).compile_pillar() - os.remove(fn_) + try: + os.remove(fn_) + except OSError: + pass self.functions, self.returners = self.__load_modules() def tune_in(self): From feed5853809c250d2550ef13570b542d79c3e467 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 09:19:42 -0700 Subject: [PATCH 420/598] Fix some trailing whitespace in the ssh module and state --- salt/modules/ssh.py | 8 ++++---- salt/states/ssh_auth.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 3bb546ae3b67..b9d529d25612 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -206,8 +206,8 @@ def set_auth_key_from_file( CLI Example:: - salt '*' ssh.set_auth_key_from_file <user> salt://ssh_keys/<user>.id_rsa.pub - ''' + salt '*' ssh.set_auth_key_from_file <user> salt://ssh_keys/<user>.id_rsa.pub + ''' # TODO: add support for pulling keys from other file sources as well lfile = __salt__['cp.cache_file'](source) if not os.path.isfile(lfile): @@ -219,7 +219,7 @@ def set_auth_key_from_file( for k in newkey.keys(): rval += set_auth_key(user, k, newkey[k]['enc'], newkey[k]['comment'], newkey[k]['options'], config) # Due to the ability for a single file to have multiple keys, it's possible for a single call - # to this function to have both "replace" and "new" as possible valid returns. I ordered the + # to this function to have both "replace" and "new" as possible valid returns. I ordered the # following as I thought best. if 'fail' in rval: return 'fail' @@ -281,7 +281,7 @@ def set_auth_key( os.makedirs(dpath) os.chown(dpath, uinfo['uid'], uinfo['gid']) os.chmod(dpath, 448) - + if not os.path.isfile(fconfig): open(fconfig, 'a+').write('{0}'.format(auth_line)) os.chown(fconfig, uinfo['uid'], uinfo['gid']) diff --git a/salt/states/ssh_auth.py b/salt/states/ssh_auth.py index d769e0a4139c..3506694cdb4c 100755 --- a/salt/states/ssh_auth.py +++ b/salt/states/ssh_auth.py @@ -45,8 +45,8 @@ def present( The comment to be placed with the ssh public key source - The source file for the key(s). Can contain any number of public keys, - in standard "authorized_keys" format. If this is set, comment, enc, + The source file for the key(s). Can contain any number of public keys, + in standard "authorized_keys" format. If this is set, comment, enc, and options will be ignored. options From a5631f9a654e7904dd4d3652a2d1858f7f27808a Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 09:21:40 -0700 Subject: [PATCH 421/598] Gracefully disable the mysql module if MySQLdb isn't available --- salt/modules/mysql.py | 73 ++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index 9b16b9ffd219..a1a39a131544 100755 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -14,11 +14,17 @@ You can also use a defaults file:: mysql.default_file: '/etc/mysql/debian.cnf' + +Required python modules: MySQLdb ''' import logging -import MySQLdb -import MySQLdb.cursors +try: + import MySQLdb + import MySQLdb.cursors + has_mysqldb = True +except ImportError: + has_mysqldb = True log = logging.getLogger(__name__) __opts__ = {} @@ -28,7 +34,8 @@ def __virtual__(): Only load this module if the mysql config is set ''' if any(k.startswith('mysql.') for k in __opts__.keys()): - return 'mysql' + if has_mysqldb: + return 'mysql' return False @@ -40,8 +47,8 @@ def __check_table(name, table): cur.execute(query) results = cur.fetchall() log.debug(results) - return results - + return results + def __repair_table(name, table): db = connect() cur = db.cursor(MySQLdb.cursors.DictCursor) @@ -51,7 +58,7 @@ def __repair_table(name, table): results = cur.fetchall() log.debug(results) return results - + def __optimize_table(name, table): db = connect() cur = db.cursor(MySQLdb.cursors.DictCursor) @@ -187,7 +194,7 @@ def free_slave(): return 'promoted' else: return 'failed' - + ''' Database related actions ''' @@ -210,7 +217,7 @@ def db_list(): log.debug(ret) return ret - + def db_tables(name): ''' Shows the tables in the given MySQL database (if exists) @@ -235,7 +242,7 @@ def db_tables(name): ret.append(table[0]) log.debug(ret) return ret - + def db_exists(name): ''' Checks if a database exists on the MySQL server. @@ -252,7 +259,7 @@ def db_exists(name): result_set = cur.fetchall() return cur.rowcount == 1 - + def db_create(name): ''' Adds a databases to the MySQL server. @@ -340,7 +347,7 @@ def user_exists(user, log.debug("Doing query: {0}".format(query,)) cur.execute( query ) return cur.rowcount == 1 - + def user_info(user, host='localhost'): ''' @@ -384,7 +391,7 @@ def user_create(user, log.debug("Query: {0}".format(query,)) cur.execute( query ) - + if user_exists(user,host): log.info("User '{0}'@'{1}' has been created".format(user,host,)) return True @@ -446,7 +453,7 @@ def user_remove(user, ''' Maintenance -''' +''' def db_check(name, table=None): ''' @@ -467,7 +474,7 @@ def db_check(name, log.info("Checking table '%s' in db '%s'..".format(name,table,)) ret = __check_table(name,table) return ret - + def db_repair(name, table=None): ''' @@ -488,7 +495,7 @@ def db_repair(name, log.info("Repairing table '%s' in db '%s'..".format(name,table,)) ret = __repair_table(name,table) return ret - + def db_optimize(name, table=None): ''' @@ -513,15 +520,15 @@ def db_optimize(name, ''' Grants ''' -def __grant_generate(grant, - database, - user, +def __grant_generate(grant, + database, + user, host='localhost', grant_option=False, escape=True): # todo: Re-order the grant so it is according to the SHOW GRANTS for xxx@yyy query (SELECT comes first, etc) grant = grant.replace(',', ', ').upper() - + db_part = database.rpartition('.') db = db_part[0] table = db_part[2] @@ -534,8 +541,8 @@ def __grant_generate(grant, query += " WITH GRANT OPTION" log.debug("Query generated: {0}".format(query,)) return query - -def user_grants(user, + +def user_grants(user, host='localhost'): ''' Shows the grants for the given MySQL user (if it exists) @@ -561,26 +568,26 @@ def user_grants(user, log.debug(ret) return ret -def grant_exists(grant, - database, - user, +def grant_exists(grant, + database, + user, host='localhost', grant_option=False, escape=True): # todo: This function is a bit tricky, since it requires the ordering to be exactly the same. # perhaps should be replaced/reworked with a better/cleaner solution. target = __grant_generate(grant, database, user, host, grant_option, escape) - + if target in user_grants(user, host): log.debug("Grant exists.") return True - + log.debug("Grant does not exist, or is perhaps not ordered properly?") return False def grant_add(grant, - database, - user, + database, + user, host='localhost', grant_option=False, escape=True): @@ -594,7 +601,7 @@ def grant_add(grant, # todo: validate grant db = connect() cur = db.cursor() - + query = __grant_generate(grant, database, user, host, grant_option, escape) log.debug("Query: {0}".format(query,)) if cur.execute( query ): @@ -602,9 +609,9 @@ def grant_add(grant, return True return False -def grant_revoke(grant, - database, - user, +def grant_revoke(grant, + database, + user, host='localhost', grant_option=False): ''' @@ -615,7 +622,7 @@ def grant_revoke(grant, salt '*' mysql.grant_revoke 'SELECT,INSERT,UPDATE' 'database.*' 'frank' 'localhost' ''' # todo: validate grant - db = connect() + db = connect() cur = db.cursor() if grant_option: From f0c54fa8845fbbbdad27d0393eaaf6c609f09d8b Mon Sep 17 00:00:00 2001 From: Eric Poelke <epoelke@gmail.com> Date: Sat, 17 Mar 2012 09:53:42 -0700 Subject: [PATCH 422/598] fix upgrade_available in yumpkg.py upgrade_available will now return the version of a package if an upgrade is present. If there is no upgrade available it will return an empty string. --- salt/modules/yumpkg.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 9578080eede5..f4f9549c5764 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -27,7 +27,6 @@ def __virtual__(): else: return False - def _list_removed(old, new): ''' List the packages which have been removed between the two package objects @@ -39,7 +38,6 @@ def _list_removed(old, new): return pkgs - def _compare_versions(old, new): ''' Returns a dict that that displays old and new versions for a package after @@ -61,7 +59,6 @@ def _compare_versions(old, new): 'new': new[npkg]} return pkgs - def available_version(name): ''' The available version of the package in the repository @@ -96,7 +93,6 @@ def available_version(name): # remove the duplicate items from the list and return the first one return list(set(versions_list))[0] - def upgrade_available(name): ''' Check whether or not an upgrade is available for a given package @@ -105,11 +101,7 @@ def upgrade_available(name): salt '*' pkg.upgrade_available <package name> ''' - if available_version(name): - return True - else: - return False - + return available_version(name) def version(name): ''' @@ -125,7 +117,6 @@ def version(name): else: return '' - def list_pkgs(*args): ''' List the packages currently installed in a dict:: @@ -150,7 +141,6 @@ def list_pkgs(*args): return pkgs - def refresh_db(): ''' Since yum refreshes the database automatically, this runs a yum clean, @@ -164,7 +154,6 @@ def refresh_db(): yb.cleanMetadata() return True - def clean_metadata(): ''' Cleans local yum metadata. @@ -232,7 +221,6 @@ def install(pkgs, refresh=False, repo='', skip_verify=False, **kwargs): return _compare_versions(old, new) - def upgrade(): ''' Run a full system upgrade, a yum upgrade @@ -263,7 +251,6 @@ def upgrade(): new = list_pkgs() return _compare_versions(old, new) - def remove(pkgs): ''' Removes packages with yum remove @@ -292,7 +279,6 @@ def remove(pkgs): return _list_removed(old, new) - def purge(pkgs): ''' Yum does not have a purge, this function calls remove From de94fd5b86f6633c43ef8d11994b8ff0f3d599e5 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 10:18:04 -0700 Subject: [PATCH 423/598] Start a bit more error checking in sysctl.assign --- salt/modules/linux_sysctl.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/salt/modules/linux_sysctl.py b/salt/modules/linux_sysctl.py index 7056fce6f3df..b197eb9c902a 100644 --- a/salt/modules/linux_sysctl.py +++ b/salt/modules/linux_sysctl.py @@ -2,6 +2,7 @@ Module for viewing and modifying sysctl parameters ''' +import re import os from salt.exceptions import CommandExecutionError @@ -62,13 +63,24 @@ def assign(name, value): if not os.path.exists(sysctl_file): raise CommandExecutionError('sysctl {0} does not exist'.format(name)) - cmd = 'sysctl -w {0}={1}'.format(name, value) - ret = {} - out = __salt__['cmd.run'](cmd).strip() - if ' = ' not in out: - raise CommandExecutionError('sysctl -w failed: {0}'.format(out)) - comps = out.split(' = ') - ret[comps[0]] = comps[1] + ret = {} + cmd = 'sysctl -w {0}={1}'.format(name, value) + data = __salt__['cmd.run_all'](cmd) + out = data['stdout'] + + # Example: + # # sysctl -w net.ipv4.tcp_rmem="4096 87380 16777216" + # net.ipv4.tcp_rmem = 4096 87380 16777216 + regex = re.compile('^{0}\s+=\s+{1}$'.format(name, value)) + + if not regex.match(out): + if data['retcode'] != 0 and data['stderr']: + error = out['stderr'] + else: + error = out['stdout'] + raise CommandExecutionError('sysctl -w failed: {0}'.format(error)) + new_name, new_value = out.split(' = ') + ret[new_name] = new_value return ret From b6645c66c85a763fe574379b4a8a9e0de083a3cc Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 10:27:00 -0700 Subject: [PATCH 424/598] Fix a stupid IndexError in my last patch --- salt/modules/linux_sysctl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/linux_sysctl.py b/salt/modules/linux_sysctl.py index b197eb9c902a..9d71811e577c 100644 --- a/salt/modules/linux_sysctl.py +++ b/salt/modules/linux_sysctl.py @@ -75,9 +75,9 @@ def assign(name, value): if not regex.match(out): if data['retcode'] != 0 and data['stderr']: - error = out['stderr'] + error = data['stderr'] else: - error = out['stdout'] + error = out raise CommandExecutionError('sysctl -w failed: {0}'.format(error)) new_name, new_value = out.split(' = ') ret[new_name] = new_value From cb4010faa6cb0d5a194ba7aed2a9ce5674f662df Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 10:18:43 -0700 Subject: [PATCH 425/598] Don't blow up in sysctl.assign on sysctl with multiple values --- salt/modules/linux_sysctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/linux_sysctl.py b/salt/modules/linux_sysctl.py index 9d71811e577c..679967aa9b31 100644 --- a/salt/modules/linux_sysctl.py +++ b/salt/modules/linux_sysctl.py @@ -64,7 +64,7 @@ def assign(name, value): raise CommandExecutionError('sysctl {0} does not exist'.format(name)) ret = {} - cmd = 'sysctl -w {0}={1}'.format(name, value) + cmd = 'sysctl -w {0}="{1}"'.format(name, value) data = __salt__['cmd.run_all'](cmd) out = data['stdout'] From 61170e3ec25cbfc30f4a2be4a95d2ceadb2248a7 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 10:53:14 -0700 Subject: [PATCH 426/598] Add some more troubleshooting documentation - Fix a few commands that were already there and rendered weird - Add a section for ports the salt master needs option - Added a section on troubleshooting a hung master --- doc/topics/troubleshooting/index.rst | 72 ++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/doc/topics/troubleshooting/index.rst b/doc/topics/troubleshooting/index.rst index bd0540635b83..73789364c33e 100644 --- a/doc/topics/troubleshooting/index.rst +++ b/doc/topics/troubleshooting/index.rst @@ -13,10 +13,28 @@ A great deal of information is available via the debug logging system, if you are having issues with minions connecting or not starting run the minion and/or master in the foreground: +.. code-block:: sh + # salt-master -l debug # salt-minion -l debug +What Ports do the Master and Minion Need Open? +============================================== + +No ports need to be opened up on each minion. For the master, tcp ports 4505 +and 4506 need to be open. If you've put your salt master and minion both in +debug mode and don't see an acknowledgement that your minion has connected, +it could very well be a firewall. + +You can check port connectivity from the minion with the nc command: + +.. code-block:: sh + + # nc -v -z salt.master.ip 4505 + # nc -v -z salt.master.ip 4506 + + Using salt-call =============== @@ -37,6 +55,8 @@ The salt-master needs at least 2 sockets per host that connects to it, one for the Publisher and one for response port. Thus, large installations may upon scaling up the number of minions accessing a given master, encounter: +.. code-block:: sh + 12:45:29,289 [salt.master ][INFO ] Starting Salt worker process 38 Too many open files sock != -1 (tcp_listener.cpp:335) @@ -44,11 +64,57 @@ scaling up the number of minions accessing a given master, encounter: The solution to this would be to check the number of files allowed to be opened by the user running salt-master (root by default): - [root@salt-master ~]# ulimit -n - 1024 +.. code-block:: sh + + [root@salt-master ~]# ulimit -n + 1024 And modify that value to be at least equal to the number of minions x 2. This setting can be changed in limits.conf as the nofile value(s), and activated upon new a login of the specified user. -So, an environment with 1800 minions, would need 1800 x 2 = 3600 as a minimum +So, an environment with 1800 minions, would need 1800 x 2 = 3600 as a minimum. + + +Salt Master Stops Responding +============================ + +There are known bugs with ZeroMQ less than 2.1.11 which can cause the salt +master to not respond properly. If you're running ZeroMQ greater than or equal +to 2.1.9, you can work around the bug by setting the sysctl net.core.rmem_max +and net.core.wmem_max to 16777216. Next set the third field in net.ipv4.tcp_rmem +and net.ipv4.tcp_wmem to at least 16777216. + +You can do it manually with something like: + +.. code-block:: sh + + # echo 16777216 > /proc/sys/net/core/rmem_max + # echo 16777216 > /proc/sys/net/core/wmem_max + # echo "4096 87380 16777216" > /proc/sys/net/ipv4/tcp_rmem + # echo "4096 87380 16777216" > /proc/sys/net/ipv4/tcp_wmem + +Or with the following salt state: + +.. code-block:: yaml + :linenos: + + net.core.rmem_max: + sysctl: + - present + - value: 16777216 + + net.core.wmem_max: + sysctl: + - present + - value: 16777216 + + net.ipv4.tcp_rmem: + sysctl: + - present + - value: 4096 87380 16777216 + + net.ipv4.tcp_wmem: + sysctl: + - present + - value: 4096 87380 16777216 From e75a6a44b805cb5910394aabc6dd30b143e7191b Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Sat, 17 Mar 2012 14:24:15 -0400 Subject: [PATCH 427/598] add get_macs() to virt.py - return a list of macs from a vm --- salt/modules/virt.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 900b1d66e163..1983d8101770 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -173,6 +173,22 @@ def node_info(): 'sockets': raw[5]} return info +def get_macs(vm_): + ''' + Return a list off MAC addresses from the named vm + + CLI Example:: + + salt '*' virt.get_macs <vm name> + ''' + macs = [] + doc = minidom.parse(StringIO.StringIO(get_xml(vm_))) + for node in doc.getElementsByTagName("devices"): + i_nodes = node.getElementsByTagName("interface") + for i_node in i_nodes: + for v_node in i_node.getElementsByTagName('mac'): + macs.append(v_node.getAttribute('address')) + return macs def get_graphics(vm_): ''' From 50368d7d1d91e92a1e79fea63556ceca1cfb873d Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Sat, 17 Mar 2012 14:25:05 -0400 Subject: [PATCH 428/598] add get_nics() to virt.py - return a dictionary of network interfaces from a vm --- salt/modules/virt.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 1983d8101770..d7b2c68226bc 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -173,6 +173,45 @@ def node_info(): 'sockets': raw[5]} return info +def get_nics(vm_): + ''' + Return info about the network interfaces of a named vm + + CLI Example:: + + salt '*' virt.get_nics <vm name> + ''' + nics = {} + doc = minidom.parse(StringIO.StringIO(get_xml(vm_))) + for node in doc.getElementsByTagName("devices"): + i_nodes = node.getElementsByTagName("interface") + for i_node in i_nodes: + nic = {} + nic['type'] = i_node.getAttribute('type') + for v_node in i_node.getElementsByTagName('*'): + if v_node.tagName == "mac": + nic['mac'] = v_node.getAttribute('address') + if v_node.tagName == "model": + nic['model'] = v_node.getAttribute('type') + # driver, source, and match can all have optional attributes + if re.match('(driver|source|address)', v_node.tagName): + temp = {} + for key in v_node.attributes.keys(): + temp[key] = v_node.getAttribute(key) + nic[str(v_node.tagName)] = temp + # virtualport needs to be handled separately, to pick up the + # type attribute of the virtualport itself + if v_node.tagName == "virtualport": + temp = {} + temp['type'] = v_node.getAttribute('type') + for key in v_node.attributes.keys(): + temp[key] = v_node.getAttribute(key) + nic['virtualport'] = temp + if 'mac' not in nic: + continue + nics[nic['mac']] = nic + return nics + def get_macs(vm_): ''' Return a list off MAC addresses from the named vm From 62b1b48dd5a35d56d0b5f5dda55d771a075274d6 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 17 Mar 2012 13:37:22 -0600 Subject: [PATCH 429/598] Add permission management to file.recurse --- salt/states/file.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index 7aa940ce84fc..21c18a87de31 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -341,6 +341,11 @@ def _check_perms(name, ret, user, group, mode): ''' Check the permissions on files and chown if needed ''' + if not ret: + ret = {'name': name, + 'changes': {}, + 'comment': '', + 'result': False} # Check permissions perms = {} perms['luser'] = __salt__['file.get_user'](name) @@ -950,6 +955,10 @@ def recurse(name, source, clean=False, require=None, + user=None, + group=None, + dir_mode=None, + file_mode=None, __env__='base'): ''' Recurse through a subdirectory on the master and copy said subdirecory @@ -985,6 +994,7 @@ def recurse(name, return _error( ret, 'The path {0} exists and is not a directory'.format(name)) os.makedirs(name) + vdir = set() for fn_ in __salt__['cp.cache_dir'](source, __env__): if not fn_.strip(): continue @@ -999,9 +1009,20 @@ def recurse(name, ) ) ) - if not os.path.isdir(os.path.dirname(dest)): + dirname = os.path.dirname(dest) + if not os.path.isdir(dirname): _makedirs(dest) + if not dirname in vdir: + # verify the directory perms if they are set + # _check_perms(name, ret, user, group, mode) + _ret, perms = _check_perms(dirname, {}, user, group, dir_mode) + if _ret['changes']: + ret['changes'][dirname] = 'updated' + vdir.add(dirname) if os.path.isfile(dest): + _ret, perms = _check_perms(dest, {}, user, group, file_mode) + if _ret['changes']: + ret['changes'][dest] = 'updated' keep.add(dest) srch = '' dsth = '' From b3d5ea9e25438b6bedd52a98efc4d9afdc38c2e2 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 17 Mar 2012 14:26:53 -0600 Subject: [PATCH 430/598] Add warning if the master is running and zeromq is too old This should fix #891, let me know if we feel that more is required --- salt/utils/verify.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 9b0c4d8d754e..7f78b6951a63 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -15,7 +15,9 @@ def zmq_version(): - '''ZeroMQ python bindings >= 2.1.9 are required''' + ''' + ZeroMQ python bindings >= 2.1.9 are required + ''' import zmq ver = zmq.__version__ # The last matched group can be None if the version @@ -50,7 +52,11 @@ def zmq_version(): return True # If all else fails, gracefully croak and warn the user - log.critical("ZeroMQ python bindings >= 2.1.9 are required") + log.critical('ZeroMQ python bindings >= 2.1.9 are required') + if 'salt-master' in sys.argv[0]: + log.critical('The Salt Master is unstable using a ZeroMQ version ' + 'lower than 2.1.11 and requires this fix: http://lists.zeromq.' + 'org/pipermail/zeromq-dev/2011-June/012094.html') return False From 939664767be8cfa536a35894fb2c53d7692c5de5 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 13:32:11 -0700 Subject: [PATCH 431/598] Gracefully catch ImportErrors on hosts lacking yum modules --- salt/modules/yumpkg.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index f4f9549c5764..cba953456694 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -1,10 +1,17 @@ ''' Support for YUM + +Required python modules: yum, rpm, rpmUtils ''' -import yum -import rpm +try: + import yum + import rpm + from rpmUtils.arch import getBaseArch + has_yumdeps = True +except ImportError: + has_yumdeps = False + import logging -from rpmUtils.arch import getBaseArch log = logging.getLogger(__name__) @@ -12,6 +19,9 @@ def __virtual__(): ''' Confine this module to yum based systems ''' + if not has_yumdeps: + return False + # Return this for pkg on RHEL/Fedora based distros that ship with python # 2.6 or greater. dists = ('CentOS', 'Scientific', 'RedHat') @@ -73,7 +83,7 @@ def available_version(name): # here we can, but for now its exact match only. versions_list = [] for pkgtype in ['available', 'updates']: - + pl = yb.doPackageLists(pkgtype) exactmatch, matched, unmatched = yum.packages.parsePackages(pl, [name]) # build a list of available packages from either available or updates From f64de477fc3f6e2f7999443f16f8eef997277c0e Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 17 Mar 2012 14:47:21 -0600 Subject: [PATCH 432/598] Add upgrade data Fix #922 --- doc/topics/releases/0.9.8.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index b45669b2d4c8..29349e80315c 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -32,6 +32,25 @@ but available in a clean and simple way. The new batch size capability allows for executions to be rolled through the targeted minions. +Upgrade Considerations +====================== + +Upgrade Issue +------------- + +There was an oversight that has been previously missed which could cause a +newer minion to crash an older master. This oversight has been repaired so +that this version incompatibility issue will not occur again. When upgrading +to 0.9.8 make sure to upgrade the master first, followed by the minions. + +Debian/Ubuntu Packages +---------------------- + +The original Debian/Ubuntu packages were called salt and included all salt +applications. New packages in the ppa are split. If an old salt package is +installed then it should be manually removed and the new split packages +need to be installed fresh. + Major Features ============== From aec0d96fdc393d1a4d0a8d5b40108ec83b656760 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 13:59:45 -0700 Subject: [PATCH 433/598] Updating release notes --- doc/topics/releases/0.9.8.rst | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index b45669b2d4c8..86086783eba6 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -6,9 +6,9 @@ Salt 0.9.8 is a big step forward, with many additions and enhancements, as well as a number of precursors to advanced future developments. This version of Salt adds much more power to the command line, making the -old hard timeout issues a thing of the past, and adding kwargs support to -salt calls. These additions are also available in the salt client api, making -the available api tools much more powerful. +old hard timeout issues a thing of the past, and adding keyword argument +support to salt calls. These additions are also available in the salt client +api, making the available api tools much more powerful. The new pillar system allows for data to be stored on the master and assigned to minions in a granular way similar to the state system. @@ -19,18 +19,20 @@ system has been added to allow for redirecting what modules function in the background for individual states. Support for OpenSUSE has been added and support for Solaris has begun -serious development. +serious development. Windows support has been signifigantly enhanced as well. -The matcher and target systems have received a great deal of attention, the -default behavior of grain matching has changed slightly to reflect the regular -salt default behavior and the compound matcher system has been greatly refined. +The matcher and target systems have received a great deal of attention. The +default behavior of grain matching has changed slightly to reflect the salt +default behavior and the compound matcher system has been greatly refined. A number of impressive features with keyword arguments have been added to both the cli and to the state system, making states much more powerful and flexible, but available in a clean and simple way. The new batch size capability allows for executions to be rolled through the -targeted minions. +targeted minions in percentage or specific numbers of minions. This was added +to prevent the "thundering herd" problem when targetting large numbers of +minions for things like service restarts or file downloads. Major Features @@ -42,7 +44,7 @@ Pillar Pillar offers an interface to declare variable data on the master that is then assigned to the minions. The pillar data is made available to all modules, states, sls files etc. It is compiled on the master and is declared using the -existing render system. This means that learning pillar should be fairly +existing renderer system. This means that learning pillar should be fairly trivial to those already familiar with salt states. CLI Additions @@ -83,3 +85,18 @@ finished for 0.9.9. The goal is that states can be compiled on both the master and the minion allowing for compilation to be split between master and minion. + + +Solaris Support +-------------------- + +Salt 0.9.8 sees the introduction of basic Solaris support. + + +Windows Support +-------------------- + +Salt states on windows are now much more viable thanks to contributions from +our community! States for file, user, and group management are more fully +fleshed out along with a network module. Windows users can also now manage +registry entries using the new "reg" module. From 81c181c33fc773192fe69d106f4a9bbd31fbcb23 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 14:14:57 -0700 Subject: [PATCH 434/598] Module updates to the release notes --- doc/topics/releases/0.9.8.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 86086783eba6..f0fd0135fa4b 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -71,6 +71,30 @@ Requisite Glob Matching Batch Size ---------- + +Module Updates +--------------- + +This is a list of notable, but not complete updates with new and existing +modules. + +For our ruby users, new 'rvm' and 'gem' modules have been added along with +the associated states. + +The 'virt' module gained basic Xen support. + +The 'pkg' module supports Scientific Linux. + +The 'pkg' module on Debian, Ubuntu, and derivatives forces apt to run in a +non-interactive mode preventing previous issues with installation. Also, +Ubuntu users gained a new upstart 'service' module. + +The 'mysql.user' state and 'mysql' module gained a *password_hash* argument. + +The 'cmd' module and state gained a *shell* keyword argument for specifying +a shell other than /bin/sh on Linux / Unix systems. + + In Progress Development ======================= From 15fb8c4c9da92324f5b5a8840952eb4118f5790c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 17 Mar 2012 15:22:50 -0600 Subject: [PATCH 435/598] Add more to the release document --- doc/topics/releases/0.9.8.rst | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 29349e80315c..38429b303bb7 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -67,15 +67,29 @@ trivial to those already familiar with salt states. CLI Additions ------------- -Better State Return Data ------------------------- +The salt command in 0.9.8 has received a serious overhaul and is more powerful +then ever. Data is returned to the terminal as it is received, and the salt +command will now wait for all running minions to return data before stopping. Kwargs and States ----------------- +A new addition to writing state modules has been made available. State modules +can now accept the ``**kwargs`` argument. The result of this is that all data +in an sls file assigned to a state will be made available to the state function. + +This enables data to be passed in a transparent way back to the modules that +are executing the logic. In particular this allows us to add arguments to the +pkg.install module that enable more advanced and granular controls with respect +to what the state is capable of. + Kwargs and the CLI ------------------ +In the past it was required that all arguments be passed in order to modules +from the salt and salt-call commands. As of 0.9.8 keyword arguments can be +passed in the form of ``kwarg=argument``. + Matcher Refinements and Changes ------------------------------- From d5cbf92089d6eb86ac4d1cb47252e6152f5e0ef2 Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Sat, 17 Mar 2012 17:27:25 -0400 Subject: [PATCH 436/598] remove localfqdn grain --- salt/grains/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 9f25161bfabf..1ccb4541f102 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -461,7 +461,6 @@ def hostname(): comps = grains['fqdn'].split('.') grains['host'] = comps[0] grains['localhost'] = socket.gethostname() - grains['localfqdn'] = __salt__['cmd.run']('hostname -f').strip() if len(comps) > 1: grains['domain'] = '.'.join(comps[1:]) else: From fba208ab94f8d2fa145e7e56c407c26b7d3c6c9b Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 17 Mar 2012 15:31:04 -0600 Subject: [PATCH 437/598] Change grain pcre matcher in compound commands to use P --- salt/minion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index 7b9eae98c22d..2cf84797e694 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -661,7 +661,7 @@ def compound_match(self, tgt): log.debug('Compound target received that is not a string') return False ref = {'G': 'grain', - 'R': 'grain_pcre', + 'P': 'grain_pcre', 'X': 'exsel', 'L': 'list', 'E': 'pcre'} From b4bd2d5a4ab0f67b302ecc80a0baf65b652b54f7 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 17 Mar 2012 15:58:54 -0600 Subject: [PATCH 438/598] more additions on the 0.9.8 release notes --- doc/topics/releases/0.9.8.rst | 51 +++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index af703a532655..6aaed21c5ee2 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -95,15 +95,66 @@ passed in the form of ``kwarg=argument``. Matcher Refinements and Changes ------------------------------- +A number of fixes and changes have been applied to the Matcher system. The +most noteworthy is the change in the grain matcher. The grain matcher used +a regular expression to match the passed data to a grain, but now defaults +to a shell glob like the majority of match interfaces in Salt. A new option +is available that still uses the old style regex matching to grain data called +grain-pcre. To use regex matching in compound matches use the letter P. + Providers --------- +Salt predetermines what modules should be mapped to what uses based on the +properties of a system. These determinations are generally made for modules +that provide things like package and service management. + +Sometimes in states it may be needed for an alternative module to be used +to provide the functionality needed. For instance, an Arch Linux system may +have been set up with systemd support, so instead of using the default service +module detected for Arch Linux, the systemd module can be used: + +.. code-block:: yaml + + http: + service: + - running + - enable: True + - provider: systemd + Requisite Glob Matching ----------------------- +Requisites can now be defined with glob expansion. This means that if there are +many requisities they can be defined in a single line. For instance, if all +files in a directory are required or watched, then it can be defined like so: + +.. code-block:: yaml + + http: + service: + - running + - enable: True + - watch: + - file: /etc/http/conf.d/* + +This example will watch all defined files that match the glob +``/etc/http/conf.d/*`` + Batch Size ---------- +The new batch size option allows for commands to be executed while maintaining +that only so many hosts are executing the command at one time. This option can +take a percentage or a finite number: + +.. code-block:: bash + + salt \* -b %10 test.ping + +This will only run test.ping on %10 of targeted systems at a time and work +through all targeted systems until the task is complete + Module Updates --------------- From 1fbde66303843ba1321d6b4ff2714786e3af5906 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 15:12:36 -0700 Subject: [PATCH 439/598] Add upstart to the list of all modules --- doc/ref/modules/all/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index f4e2ff251b6f..f48da7926968 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -56,6 +56,7 @@ Full list of builtin modules systemd test tomcat + upstart useradd virt virtualenv From d79b9f96528b7ae49d42c2b612bc6a7064c7b9c4 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 15:21:17 -0700 Subject: [PATCH 440/598] Document the "password_hash" kwarg to mysql.user --- salt/modules/mysql.py | 10 +++++++--- salt/states/mysql_user.py | 8 +++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index a1a39a131544..36e22890d1ab 100755 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -373,9 +373,11 @@ def user_create(user, ''' Creates a MySQL user. - CLI Example:: + CLI Examples:: + + salt '*' mysql.user_create 'username' 'hostname' 'password - salt '*' mysql.user_create 'username' 'hostname' 'password' + salt '*' mysql.user_create 'username' 'hostname' password_hash='hash' ''' if user_exists(user,host): log.info("User '{0}'@'{1}' already exists".format(user,host,)) @@ -406,9 +408,11 @@ def user_chpass(user, ''' Change password for MySQL user - CLI Example:: + CLI Examples:: salt '*' mysql.user_chpass frank localhost newpassword + + salt '*' mysql.user_chpass frank localhost password_hash='hash' ''' if password is None or password_hash is None: log.error('No password provided') diff --git a/salt/states/mysql_user.py b/salt/states/mysql_user.py index 81f09aba941e..f739f18744d8 100644 --- a/salt/states/mysql_user.py +++ b/salt/states/mysql_user.py @@ -22,7 +22,13 @@ def present(name, name The name of the user to manage - ''' + + password + The password + + password_hash + The password in hashed form + ''' ret = {'name': name, 'changes': {}, 'result': True, From 4eed2d01351cc186179999698da00a3c7a96adf3 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 15:21:40 -0700 Subject: [PATCH 441/598] Add rvm and gem modules to the all modules doc index --- doc/ref/modules/all/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index f48da7926968..b2d997708030 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -24,6 +24,7 @@ Full list of builtin modules file freebsdkmod freebsdpkg + gem gentoo_service grains groupadd @@ -44,6 +45,7 @@ Full list of builtin modules pw_group pw_user rh_service + rvm saltutil selinux service From 40a4edecf4deaa3647627c92bdf458ca7a41e2d6 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 15:24:38 -0700 Subject: [PATCH 442/598] s/cmd/cmdmod/ in all modules docs --- doc/ref/modules/all/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index b2d997708030..09e76c02a1e7 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -15,7 +15,7 @@ Full list of builtin modules archive butterkvm cluster - cmd + cmdmod cp cron data From 9b206a623319eaa2188f9abc5a10fb6413f02ade Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 15:28:49 -0700 Subject: [PATCH 443/598] Adding gem and rvm states to the state index --- doc/ref/states/all/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/ref/states/all/index.rst b/doc/ref/states/all/index.rst index eff9f1ffeb88..6ba781218b61 100644 --- a/doc/ref/states/all/index.rst +++ b/doc/ref/states/all/index.rst @@ -13,6 +13,7 @@ Full list of builtin states cmd cron file + gem group host kmod @@ -20,6 +21,7 @@ Full list of builtin states mysql_database mysql_user pkg + rvm service ssh_auth sysctl From 635e0c9a20a1802dd85155540fa6356bf792b2e7 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 15:32:32 -0700 Subject: [PATCH 444/598] Fairly large doc update for the 0.9.8 release notes --- doc/topics/releases/0.9.8.rst | 44 +++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index ae4a360da6a2..02cec0db0176 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -81,6 +81,29 @@ Kwargs and the CLI Matcher Refinements and Changes ------------------------------- +Support has been added for matching minions with Yahoo's range library. This +is handled by passing range syntax with *-R* or *--range* arguments to salt. + +More information at: +https://github.com/grierj/range/wiki/Introduction-to-Range-with-YAML-files + +Much like the rest of salt, grains are now matched by default using globs +instead of regular expressions. A new *--grain-pcre* was added to the salt +command line for doing just this: + +.. code-block:: sh + + # salt --grain-pcre 'os:(Arch:Fed).*' test.ping + +The associated compound matcher for use in states or the ``top.sls`` is *P*: + + +.. code-block:: sh + + P@os:(Arch|Fed).* + +**NOTE**: The default grains matcher has changed from pcre to glob for consistency + Providers --------- @@ -97,21 +120,22 @@ Module Updates This is a list of notable, but not complete updates with new and existing modules. -For our ruby users, new 'rvm' and 'gem' modules have been added along with -the associated states. +For our ruby users, new :doc:`rvm </ref/modules/all/salt.modules.rvm>` and :doc:`gem </ref/modules/all/salt.modules.gem>` modules have been added along with +the :doc:`associated </ref/states/all/salt.states.rvm>` :doc:`states </ref/states/all/salt.states.gem>` + -The 'virt' module gained basic Xen support. +The :doc:`virt </ref/modules/all/salt.modules.virt>` module gained basic Xen support. -The 'pkg' module supports Scientific Linux. +The :doc:`pkg </ref/modules/all/salt.modules.yumpkg>` module gained Scientific Linux support. -The 'pkg' module on Debian, Ubuntu, and derivatives forces apt to run in a -non-interactive mode preventing previous issues with installation. Also, -Ubuntu users gained a new upstart 'service' module. +The :doc:`pkg </ref/modules/all/salt.modules.apt>` module on Debian, Ubuntu, and derivatives +forces apt to run in a non-interactive mode preventing previous issues with installation. Also, +Ubuntu users gained a new upstart :doc:`service </ref/modules/all/salt.modules.upstart>` module. -The 'mysql.user' state and 'mysql' module gained a *password_hash* argument. +The :doc:`mysql.user </ref/states/all/salt.states.mysql_user>` state and :doc:`mysql </ref/modules/all/salt.modules.mysql>` module gained a *password_hash* argument. -The 'cmd' module and state gained a *shell* keyword argument for specifying -a shell other than /bin/sh on Linux / Unix systems. +The :doc:`cmd </ref/modules/all/salt.modules.cmdmod>` module and state gained a *shell* keyword argument for specifying +a shell other than ``/bin/sh`` on Linux / Unix systems. In Progress Development From 3f641809cf667c9a1df3f3d05b92fdffa4004f16 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sat, 17 Mar 2012 22:34:23 +0000 Subject: [PATCH 445/598] adding version number to changes dict return --- salt/states/pip.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/states/pip.py b/salt/states/pip.py index 4193b62c32fd..74e97edf06ec 100644 --- a/salt/states/pip.py +++ b/salt/states/pip.py @@ -83,8 +83,11 @@ def installed(name, no_install=no_install, no_download=no_download, install_options=install_options): + pkg_list = __salt__['pip.list'](name, bin_env) + version = pkg_list.values()[0] + pkg_name = pkg_list.keys()[0] ret['result'] = True - ret['changes'][name] = 'Installed' + ret['changes']["{0}=={1}".format(pkg_name, version)] = 'Installed' ret['comment'] = 'Package was successfully installed' else: ret['result'] = False From 3ab80337bfd74e94f19e90091d0d765540ac8255 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sat, 17 Mar 2012 22:38:08 +0000 Subject: [PATCH 446/598] changing pip_bin to bin_env argument to keep with new convetion --- salt/states/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/pip.py b/salt/states/pip.py index 9777936b72cd..503d545b8a52 100644 --- a/salt/states/pip.py +++ b/salt/states/pip.py @@ -58,7 +58,7 @@ def installed(name, bin_env = env ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} - if name in __salt__['pip.list'](name, pip_bin): + if name in __salt__['pip.list'](name, bin_env): ret['result'] = True ret['comment'] = 'Package already installed' return ret From be43260ffa722fc37c384f115cafa065e865a26f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 17 Mar 2012 16:53:30 -0600 Subject: [PATCH 447/598] Make salt-call honor the no-color option correctly --- salt/cli/caller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/cli/caller.py b/salt/cli/caller.py index 2d789fe46177..2fd33092ff7d 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -117,5 +117,5 @@ def run(self): printout = self._get_outputter() if 'json_out' in self.opts and self.opts['json_out']: printout.indent = 2 - - printout({'local': ret['return']}, color=self.opts['no_color']) + color = not bool(self.opts['no_color']) + printout({'local': ret['return']}, color=color) From ed053eb1dd74778c2d5861fec1f8786e67373e5d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sat, 17 Mar 2012 17:01:36 -0600 Subject: [PATCH 448/598] Turn on colored output by default and enable it for the salt cmd --- salt/cli/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 928ea59ee481..00ab7a731f65 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -179,6 +179,11 @@ def __parse(self): action='store_true', dest='json_out', help='Print the output from the salt command in json.') + parser.add_option('--no-color', + default=False, + action='store_true', + dest='no_color', + help='Disable all colored output') options, args = parser.parse_args() @@ -336,7 +341,8 @@ def _output_ret(self, ret, out): # Pretty print any salt exceptions elif isinstance(ret, SaltException): printout = get_outputter("txt") - printout(ret) + color = not bool(self.opts['no_color']) + printout(ret, color=color) def _format_ret(self, full_ret): ''' From 0df3566de8bab46b9dd161fa534765616a3f1a09 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 16:27:56 -0700 Subject: [PATCH 449/598] More updates to the 0.9.8 release notes --- doc/topics/releases/0.9.8.rst | 62 ++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index e346fa02a6c5..2eda5f4a58b3 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -37,8 +37,8 @@ minions for things like service restarts or file downloads. Upgrade Considerations ====================== -Upgrade Issue -------------- +Upgrade Issues +-------------- There was an oversight that has been previously missed which could cause a newer minion to crash an older master. This oversight has been repaired so @@ -72,25 +72,26 @@ CLI Additions The salt command in 0.9.8 has received a serious overhaul and is more powerful then ever. Data is returned to the terminal as it is received, and the salt command will now wait for all running minions to return data before stopping. +This makes adding very large *--timeout* arguments completely unnecessary. -Kwargs and States +Keyword Arguments and States ----------------- A new addition to writing state modules has been made available. State modules can now accept the ``**kwargs`` argument. The result of this is that all data -in an sls file assigned to a state will be made available to the state function. +in a sls file assigned to a state will be made available to the state function. This enables data to be passed in a transparent way back to the modules that -are executing the logic. In particular this allows us to add arguments to the +are executing the logic. In particular, this allows us to add arguments to the pkg.install module that enable more advanced and granular controls with respect to what the state is capable of. -Kwargs and the CLI +Keyword Arguments and the CLI ------------------ In the past it was required that all arguments be passed in order to modules -from the salt and salt-call commands. As of 0.9.8 keyword arguments can be -passed in the form of ``kwarg=argument``. +from the *salt* and *salt-call* commands. As of 0.9.8, keyword arguments can +be passed in the form of ``kwarg=argument``. Matcher Refinements and Changes ------------------------------- @@ -101,29 +102,27 @@ is handled by passing range syntax with *-R* or *--range* arguments to salt. More information at: https://github.com/grierj/range/wiki/Introduction-to-Range-with-YAML-files -Much like the rest of salt, grains are now matched by default using globs -instead of regular expressions. A new *--grain-pcre* was added to the salt -command line for doing just this: +A number of fixes and changes have been applied to the Matcher system. The +most noteworthy is the change in the grain matcher. The grain matcher used +a regular expression to match the passed data to a grain, but now defaults +to a shell glob like the majority of match interfaces in Salt. A new option +is available that still uses the old style regex matching to grain data called +grain-pcre. To use regex matching in compound matches use the letter *P*. + +For example, this would match any ArchLinux or Fedora minions: .. code-block:: sh # salt --grain-pcre 'os:(Arch:Fed).*' test.ping -The associated compound matcher for use in states or the ``top.sls`` is *P*: - +And the associated compound matcher suitable for ``top.sls`` is *P*: .. code-block:: sh P@os:(Arch|Fed).* -**NOTE**: The default grains matcher has changed from pcre to glob for consistency -======= -A number of fixes and changes have been applied to the Matcher system. The -most noteworthy is the change in the grain matcher. The grain matcher used -a regular expression to match the passed data to a grain, but now defaults -to a shell glob like the majority of match interfaces in Salt. A new option -is available that still uses the old style regex matching to grain data called -grain-pcre. To use regex matching in compound matches use the letter P. +**NOTE**: The default grains matcher has changed from pcre to glob. This is a +backwards incompatible change. Providers --------- @@ -132,8 +131,8 @@ Salt predetermines what modules should be mapped to what uses based on the properties of a system. These determinations are generally made for modules that provide things like package and service management. -Sometimes in states it may be needed for an alternative module to be used -to provide the functionality needed. For instance, an Arch Linux system may +Sometimes in states, it may be necessary for an alternative module to be used +to provide the desired functionality. For instance, an Arch Linux system may have been set up with systemd support, so instead of using the default service module detected for Arch Linux, the systemd module can be used: @@ -149,8 +148,8 @@ Requisite Glob Matching ----------------------- Requisites can now be defined with glob expansion. This means that if there are -many requisities they can be defined in a single line. For instance, if all -files in a directory are required or watched, then it can be defined like so: +many requisites, they can be defined on a single line. For instance, to watch +all files in a directory, it can be defined like so: .. code-block:: yaml @@ -167,16 +166,19 @@ This example will watch all defined files that match the glob Batch Size ---------- -The new batch size option allows for commands to be executed while maintaining -that only so many hosts are executing the command at one time. This option can +The new batch size option allows commands to be executed while maintaining that +only so many hosts are executing the command at one time. This option can take a percentage or a finite number: .. code-block:: bash - salt \* -b %10 test.ping + salt \* -b 10% test.ping + + salt -G 'os:RedHat' --batch-size 10% apache.signal restart -This will only run test.ping on %10 of targeted systems at a time and work -through all targeted systems until the task is complete +This will only run test.ping on 10% of the targeted minions at a time and work +through them all until the task is complete. This makes tasks like rolling web +server restarts behind a load balancer much easier with salt. Module Updates From 1c1ba99439d6fb7cf6442fc2d6e0a5e7b525942d Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 16:27:56 -0700 Subject: [PATCH 450/598] More updates to the 0.9.8 release notes --- doc/topics/releases/0.9.8.rst | 62 ++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index e346fa02a6c5..2eda5f4a58b3 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -37,8 +37,8 @@ minions for things like service restarts or file downloads. Upgrade Considerations ====================== -Upgrade Issue -------------- +Upgrade Issues +-------------- There was an oversight that has been previously missed which could cause a newer minion to crash an older master. This oversight has been repaired so @@ -72,25 +72,26 @@ CLI Additions The salt command in 0.9.8 has received a serious overhaul and is more powerful then ever. Data is returned to the terminal as it is received, and the salt command will now wait for all running minions to return data before stopping. +This makes adding very large *--timeout* arguments completely unnecessary. -Kwargs and States +Keyword Arguments and States ----------------- A new addition to writing state modules has been made available. State modules can now accept the ``**kwargs`` argument. The result of this is that all data -in an sls file assigned to a state will be made available to the state function. +in a sls file assigned to a state will be made available to the state function. This enables data to be passed in a transparent way back to the modules that -are executing the logic. In particular this allows us to add arguments to the +are executing the logic. In particular, this allows us to add arguments to the pkg.install module that enable more advanced and granular controls with respect to what the state is capable of. -Kwargs and the CLI +Keyword Arguments and the CLI ------------------ In the past it was required that all arguments be passed in order to modules -from the salt and salt-call commands. As of 0.9.8 keyword arguments can be -passed in the form of ``kwarg=argument``. +from the *salt* and *salt-call* commands. As of 0.9.8, keyword arguments can +be passed in the form of ``kwarg=argument``. Matcher Refinements and Changes ------------------------------- @@ -101,29 +102,27 @@ is handled by passing range syntax with *-R* or *--range* arguments to salt. More information at: https://github.com/grierj/range/wiki/Introduction-to-Range-with-YAML-files -Much like the rest of salt, grains are now matched by default using globs -instead of regular expressions. A new *--grain-pcre* was added to the salt -command line for doing just this: +A number of fixes and changes have been applied to the Matcher system. The +most noteworthy is the change in the grain matcher. The grain matcher used +a regular expression to match the passed data to a grain, but now defaults +to a shell glob like the majority of match interfaces in Salt. A new option +is available that still uses the old style regex matching to grain data called +grain-pcre. To use regex matching in compound matches use the letter *P*. + +For example, this would match any ArchLinux or Fedora minions: .. code-block:: sh # salt --grain-pcre 'os:(Arch:Fed).*' test.ping -The associated compound matcher for use in states or the ``top.sls`` is *P*: - +And the associated compound matcher suitable for ``top.sls`` is *P*: .. code-block:: sh P@os:(Arch|Fed).* -**NOTE**: The default grains matcher has changed from pcre to glob for consistency -======= -A number of fixes and changes have been applied to the Matcher system. The -most noteworthy is the change in the grain matcher. The grain matcher used -a regular expression to match the passed data to a grain, but now defaults -to a shell glob like the majority of match interfaces in Salt. A new option -is available that still uses the old style regex matching to grain data called -grain-pcre. To use regex matching in compound matches use the letter P. +**NOTE**: The default grains matcher has changed from pcre to glob. This is a +backwards incompatible change. Providers --------- @@ -132,8 +131,8 @@ Salt predetermines what modules should be mapped to what uses based on the properties of a system. These determinations are generally made for modules that provide things like package and service management. -Sometimes in states it may be needed for an alternative module to be used -to provide the functionality needed. For instance, an Arch Linux system may +Sometimes in states, it may be necessary for an alternative module to be used +to provide the desired functionality. For instance, an Arch Linux system may have been set up with systemd support, so instead of using the default service module detected for Arch Linux, the systemd module can be used: @@ -149,8 +148,8 @@ Requisite Glob Matching ----------------------- Requisites can now be defined with glob expansion. This means that if there are -many requisities they can be defined in a single line. For instance, if all -files in a directory are required or watched, then it can be defined like so: +many requisites, they can be defined on a single line. For instance, to watch +all files in a directory, it can be defined like so: .. code-block:: yaml @@ -167,16 +166,19 @@ This example will watch all defined files that match the glob Batch Size ---------- -The new batch size option allows for commands to be executed while maintaining -that only so many hosts are executing the command at one time. This option can +The new batch size option allows commands to be executed while maintaining that +only so many hosts are executing the command at one time. This option can take a percentage or a finite number: .. code-block:: bash - salt \* -b %10 test.ping + salt \* -b 10% test.ping + + salt -G 'os:RedHat' --batch-size 10% apache.signal restart -This will only run test.ping on %10 of targeted systems at a time and work -through all targeted systems until the task is complete +This will only run test.ping on 10% of the targeted minions at a time and work +through them all until the task is complete. This makes tasks like rolling web +server restarts behind a load balancer much easier with salt. Module Updates From 3bc5292979a730064663ad542270783a6f48453f Mon Sep 17 00:00:00 2001 From: blast_hardcheese <blast@hardchee.se> Date: Sat, 17 Mar 2012 16:31:50 -0700 Subject: [PATCH 451/598] Fixing some bugs in gem module - if <func>: was missing parens, always evaluated to True - _gem() can return False, so .splitlines should only be used if the return is a string --- salt/modules/gem.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/modules/gem.py b/salt/modules/gem.py index 7a626fab72be..43b281c6e79b 100644 --- a/salt/modules/gem.py +++ b/salt/modules/gem.py @@ -11,7 +11,7 @@ def _gem(command, ruby=None, runas=None): cmdline = "gem {command}".format(command=command) - if __salt__["rvm.is_installed"]: + if __salt__["rvm.is_installed"](): return __salt__["rvm.do"](ruby, cmdline, runas=runas) ret = __salt__["cmd.run_all"]( @@ -92,8 +92,12 @@ def list(prefix="", ruby=None, runas=None): The user to run gem as. """ gems = {} - for line in _gem("list {prefix}".format(prefix=prefix), - ruby, runas=runas).splitlines(): + stdout = _gem("list {prefix}".format(prefix=prefix), + ruby, runas=runas) + lines = [] + if isinstance(stdout, str): + lines = stdout.splitlines() + for line in lines: m = re.match("^([^ ]+) \((.+)\)", line) if m: gem = m.group(1) From 8cf6181494e57b06d3c521002a35692578a2b779 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 16:44:22 -0700 Subject: [PATCH 452/598] Add some new module docs and update release notes --- doc/ref/modules/all/index.rst | 8 ++++++++ .../all/{salt.modules.cmd.rst => salt.modules.cmdmod.rst} | 0 doc/ref/modules/all/salt.modules.gem.rst | 6 ++++++ doc/ref/modules/all/salt.modules.rvm.rst | 6 ++++++ doc/ref/modules/all/salt.modules.upstart.rst | 6 ++++++ 5 files changed, 26 insertions(+) rename doc/ref/modules/all/{salt.modules.cmd.rst => salt.modules.cmdmod.rst} (100%) create mode 100644 doc/ref/modules/all/salt.modules.gem.rst create mode 100644 doc/ref/modules/all/salt.modules.rvm.rst create mode 100644 doc/ref/modules/all/salt.modules.upstart.rst diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index 09e76c02a1e7..c2e19cb5629f 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -19,24 +19,31 @@ Full list of builtin modules cp cron data + debconf disk ebuild file freebsdkmod freebsdpkg + freebsdservice gem gentoo_service + git grains groupadd + hg hosts kmod + kvm_hyper linux_sysctl mdadm moosefs mount mysql network + nginx pacman + pillar pip pkg ps @@ -67,3 +74,4 @@ Full list of builtin modules win_useradd yumpkg yumpkg5 + zypper diff --git a/doc/ref/modules/all/salt.modules.cmd.rst b/doc/ref/modules/all/salt.modules.cmdmod.rst similarity index 100% rename from doc/ref/modules/all/salt.modules.cmd.rst rename to doc/ref/modules/all/salt.modules.cmdmod.rst diff --git a/doc/ref/modules/all/salt.modules.gem.rst b/doc/ref/modules/all/salt.modules.gem.rst new file mode 100644 index 000000000000..58c1469023f6 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.gem.rst @@ -0,0 +1,6 @@ +================ +salt.modules.gem +================ + +.. automodule:: salt.modules.gem + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.rvm.rst b/doc/ref/modules/all/salt.modules.rvm.rst new file mode 100644 index 000000000000..e2bfbbd4b97a --- /dev/null +++ b/doc/ref/modules/all/salt.modules.rvm.rst @@ -0,0 +1,6 @@ +================ +salt.modules.rvm +================ + +.. automodule:: salt.modules.rvm + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.upstart.rst b/doc/ref/modules/all/salt.modules.upstart.rst new file mode 100644 index 000000000000..c4f117bfcc71 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.upstart.rst @@ -0,0 +1,6 @@ +==================== +salt.modules.upstart +==================== + +.. automodule:: salt.modules.upstart + :members: \ No newline at end of file From 565ef6b1641702e4b09fbe8a1e9d31160b2b4bf3 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 16:44:22 -0700 Subject: [PATCH 453/598] Add some new module docs and update release notes --- doc/ref/modules/all/index.rst | 8 ++++++++ .../all/{salt.modules.cmd.rst => salt.modules.cmdmod.rst} | 0 doc/ref/modules/all/salt.modules.gem.rst | 6 ++++++ doc/ref/modules/all/salt.modules.rvm.rst | 6 ++++++ doc/ref/modules/all/salt.modules.upstart.rst | 6 ++++++ 5 files changed, 26 insertions(+) rename doc/ref/modules/all/{salt.modules.cmd.rst => salt.modules.cmdmod.rst} (100%) create mode 100644 doc/ref/modules/all/salt.modules.gem.rst create mode 100644 doc/ref/modules/all/salt.modules.rvm.rst create mode 100644 doc/ref/modules/all/salt.modules.upstart.rst diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index 09e76c02a1e7..c2e19cb5629f 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -19,24 +19,31 @@ Full list of builtin modules cp cron data + debconf disk ebuild file freebsdkmod freebsdpkg + freebsdservice gem gentoo_service + git grains groupadd + hg hosts kmod + kvm_hyper linux_sysctl mdadm moosefs mount mysql network + nginx pacman + pillar pip pkg ps @@ -67,3 +74,4 @@ Full list of builtin modules win_useradd yumpkg yumpkg5 + zypper diff --git a/doc/ref/modules/all/salt.modules.cmd.rst b/doc/ref/modules/all/salt.modules.cmdmod.rst similarity index 100% rename from doc/ref/modules/all/salt.modules.cmd.rst rename to doc/ref/modules/all/salt.modules.cmdmod.rst diff --git a/doc/ref/modules/all/salt.modules.gem.rst b/doc/ref/modules/all/salt.modules.gem.rst new file mode 100644 index 000000000000..58c1469023f6 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.gem.rst @@ -0,0 +1,6 @@ +================ +salt.modules.gem +================ + +.. automodule:: salt.modules.gem + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.rvm.rst b/doc/ref/modules/all/salt.modules.rvm.rst new file mode 100644 index 000000000000..e2bfbbd4b97a --- /dev/null +++ b/doc/ref/modules/all/salt.modules.rvm.rst @@ -0,0 +1,6 @@ +================ +salt.modules.rvm +================ + +.. automodule:: salt.modules.rvm + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.upstart.rst b/doc/ref/modules/all/salt.modules.upstart.rst new file mode 100644 index 000000000000..c4f117bfcc71 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.upstart.rst @@ -0,0 +1,6 @@ +==================== +salt.modules.upstart +==================== + +.. automodule:: salt.modules.upstart + :members: \ No newline at end of file From 221044af47cec07b4de951e70c265a9c4982c848 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 16:45:49 -0700 Subject: [PATCH 454/598] More documentation fun with release notes --- doc/topics/releases/0.9.8.rst | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 2eda5f4a58b3..953a070feeb2 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -172,13 +172,14 @@ take a percentage or a finite number: .. code-block:: bash - salt \* -b 10% test.ping + salt \* -b 10% test.ping - salt -G 'os:RedHat' --batch-size 10% apache.signal restart + salt -G 'os:RedHat' --batch-size 25% apache.signal restart This will only run test.ping on 10% of the targeted minions at a time and work through them all until the task is complete. This makes tasks like rolling web -server restarts behind a load balancer much easier with salt. +server restarts behind a load balancer or doing maintenance on BSD firewalls +using carp much easier with salt. Module Updates @@ -195,16 +196,21 @@ with the :doc:`associated </ref/states/all/salt.states.rvm>` The :doc:`virt </ref/modules/all/salt.modules.virt>` module gained basic Xen support. -The :doc:`pkg </ref/modules/all/salt.modules.yumpkg>` module gained Scientific Linux support. +The :doc:`pkg </ref/modules/all/salt.modules.yumpkg>` module gained Scientific +Linux support. -The :doc:`pkg </ref/modules/all/salt.modules.apt>` module on Debian, Ubuntu, and derivatives -forces apt to run in a non-interactive mode preventing previous issues with installation. Also, -Ubuntu users gained a new upstart :doc:`service </ref/modules/all/salt.modules.upstart>` module. +The :doc:`pkg </ref/modules/all/salt.modules.apt>` module on Debian, Ubuntu, +and derivatives forces apt to run in a non-interactive mode preventing previous +issues with installation. Also, Ubuntu users gained a new upstart +:doc:`service </ref/modules/all/salt.modules.upstart>` module. -The :doc:`mysql.user </ref/states/all/salt.states.mysql_user>` state and :doc:`mysql </ref/modules/all/salt.modules.mysql>` module gained a *password_hash* argument. +The :doc:`mysql.user </ref/states/all/salt.states.mysql_user>` state and +:doc:`mysql </ref/modules/all/salt.modules.mysql>` module gained a +*password_hash* argument. -The :doc:`cmd </ref/modules/all/salt.modules.cmdmod>` module and state gained a *shell* keyword argument for specifying -a shell other than ``/bin/sh`` on Linux / Unix systems. +The :doc:`cmd </ref/modules/all/salt.modules.cmdmod>` module and state gained +a *shell* keyword argument for specifying a shell other than ``/bin/sh`` on +Linux / Unix systems. In Progress Development From 7cf08ed8d22defd2c95e19bea449bd2c246f452a Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 16:51:08 -0700 Subject: [PATCH 455/598] Rename the debconf salt module to fix conflicts >>> import debconf >>> debconf.__file__ '/usr/lib/python2.6/dist-packages/debconf.py' --- doc/ref/modules/all/index.rst | 2 +- salt/modules/{debconf.py => debconfmod.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename salt/modules/{debconf.py => debconfmod.py} (100%) diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index c2e19cb5629f..3803f7ef9524 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -19,7 +19,7 @@ Full list of builtin modules cp cron data - debconf + debconfmod disk ebuild file diff --git a/salt/modules/debconf.py b/salt/modules/debconfmod.py similarity index 100% rename from salt/modules/debconf.py rename to salt/modules/debconfmod.py From d12acd9ec5c46673ceca4c721cf724269e67b624 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 17:01:52 -0700 Subject: [PATCH 456/598] Continuing updates for 0.9.8 release notes --- doc/topics/releases/0.9.8.rst | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 953a070feeb2..08113de0213b 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -73,6 +73,9 @@ The salt command in 0.9.8 has received a serious overhaul and is more powerful then ever. Data is returned to the terminal as it is received, and the salt command will now wait for all running minions to return data before stopping. This makes adding very large *--timeout* arguments completely unnecessary. +When calling salt via sudo, the user originally running salt is saved to the +salt log for auditing purposes. The *salt-key* command gained the *-D* and +*--delete-all* arguments for removing all keys. Keyword Arguments and States ----------------- @@ -196,13 +199,23 @@ with the :doc:`associated </ref/states/all/salt.states.rvm>` The :doc:`virt </ref/modules/all/salt.modules.virt>` module gained basic Xen support. -The :doc:`pkg </ref/modules/all/salt.modules.yumpkg>` module gained Scientific +The :doc:`yum </ref/modules/all/salt.modules.yumpkg5>` +:doc:`pkg </ref/modules/all/salt.modules.yumpkg>` modules gained Scientific Linux support. The :doc:`pkg </ref/modules/all/salt.modules.apt>` module on Debian, Ubuntu, and derivatives forces apt to run in a non-interactive mode preventing previous -issues with installation. Also, Ubuntu users gained a new upstart -:doc:`service </ref/modules/all/salt.modules.upstart>` module. +issues with installation. + +A :doc:`pkg </ref/modules/all/salt.modules.zypper>` module for OpenSUSE's +zypper was added. + +The :doc:`service </ref/modules/all/salt.modules.upstart>` module on ubuntu +natively supports upstart. + +A new :doc:`debconf </ref/modules/all/salt.modules.debconfmod>` module was +contributed by our community for more advanced control over deb package +deployments on Debian based distributions. The :doc:`mysql.user </ref/states/all/salt.states.mysql_user>` state and :doc:`mysql </ref/modules/all/salt.modules.mysql>` module gained a From cc66806d6255520a0166ba4d85806c3412006e76 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 17:26:08 -0700 Subject: [PATCH 457/598] Grammar updates --- doc/topics/releases/0.9.8.rst | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 08113de0213b..f2e07aff6f1f 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -7,16 +7,18 @@ well as a number of precursors to advanced future developments. This version of Salt adds much more power to the command line, making the old hard timeout issues a thing of the past, and adding keyword argument -support to salt calls. These additions are also available in the salt client -api, making the available api tools much more powerful. +support. These additions are also available in the salt client api, making +the available api tools much more powerful. The new pillar system allows for data to be stored on the master and -assigned to minions in a granular way similar to the state system. +assigned to minions in a granular way similar to the state system. It also +allows flexibility for users who want to keep data out of their state tree +similar to 'external lookup' functionality in other tools. Additions to requisites making them much more powerful have been added and improved error checking for sls files in the state system. A new provider -system has been added to allow for redirecting what modules function in -the background for individual states. +system has been added to allow for redirecting what modules run in the +background for individual states. Support for OpenSUSE has been added and support for Solaris has begun serious development. Windows support has been signifigantly enhanced as well. @@ -26,13 +28,13 @@ default behavior of grain matching has changed slightly to reflect the salt default behavior and the compound matcher system has been greatly refined. A number of impressive features with keyword arguments have been added to both -the cli and to the state system, making states much more powerful and flexible, -but available in a clean and simple way. +the cli and to the state system. This makes states much more powerful and +flexible while maintaining the simple configuration everyone loves. -The new batch size capability allows for executions to be rolled through the -targeted minions in percentage or specific numbers of minions. This was added -to prevent the "thundering herd" problem when targetting large numbers of -minions for things like service restarts or file downloads. +The new batch size capability allows for executions to be rolled through a +group of targeted minions a percentage or specific number at a time. This +was added to prevent the "thundering herd" problem when targetting large +numbers of minions for things like service restarts or file downloads. Upgrade Considerations ====================== From f9d5705cd7586449f5c705e0e069cddd9674a226 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 17:26:33 -0700 Subject: [PATCH 458/598] More updates This one includes: - Installing packages for Ubuntu - kwarg pkg.installed state example from @blast_hardcheese - Information on new file client --- doc/topics/releases/0.9.8.rst | 94 +++++++++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 22 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index f2e07aff6f1f..1e0cb80b87d4 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -42,18 +42,41 @@ Upgrade Considerations Upgrade Issues -------------- -There was an oversight that has been previously missed which could cause a -newer minion to crash an older master. This oversight has been repaired so -that this version incompatibility issue will not occur again. When upgrading -to 0.9.8 make sure to upgrade the master first, followed by the minions. +There was a previously missed oversight which could cause a newer minion to +crash an older master. That oversight has been resolved so the version +incompatibility issue will no longer occur. When upgrading to 0.9.8 make +sure to upgrade the master first, followed by the minions. Debian/Ubuntu Packages ---------------------- The original Debian/Ubuntu packages were called salt and included all salt -applications. New packages in the ppa are split. If an old salt package is -installed then it should be manually removed and the new split packages -need to be installed fresh. +applications. New packages in the ppa are split by function. If an old salt +package is installed then it should be manually removed and the new split +packages need to be freshly installed. + +On the master: + +.. code-block:: sh + + # apt-get purge salt + # apt-get install salt-{master,minion} + +On the minions: + +.. code-block:: sh + + # apt-get purge salt + # apt-get install salt-minion + +And on any Syndics: + +.. code-block:: sh + + # apt-get install salt-syndic + +The official salt stack ppa for Ubuntu is located at: +https://launchpad.net/~saltstack/+archive/salt Major Features @@ -71,25 +94,43 @@ trivial to those already familiar with salt states. CLI Additions ------------- -The salt command in 0.9.8 has received a serious overhaul and is more powerful -then ever. Data is returned to the terminal as it is received, and the salt +The ``salt`` command has received a serious overhaul and is more powerful +than ever. Data is returned to the terminal as it is received, and the salt command will now wait for all running minions to return data before stopping. This makes adding very large *--timeout* arguments completely unnecessary. + When calling salt via sudo, the user originally running salt is saved to the -salt log for auditing purposes. The *salt-key* command gained the *-D* and -*--delete-all* arguments for removing all keys. +salt log for auditing purposes. This makes it easy to see who ran what by just +looking through the minion logs. + +The *salt-key* command gained the *-D* and *--delete-all* arguments for +removing all keys. Be careful with this one! Keyword Arguments and States ----------------- A new addition to writing state modules has been made available. State modules -can now accept the ``**kwargs`` argument. The result of this is that all data +now accept the ``**kwargs`` argument. The result of this is that all data in a sls file assigned to a state will be made available to the state function. -This enables data to be passed in a transparent way back to the modules that -are executing the logic. In particular, this allows us to add arguments to the -pkg.install module that enable more advanced and granular controls with respect -to what the state is capable of. +This passes data in a transparent way back to the modules executing the logic. +In particular, this allows adding arguments to the ``pkg.install`` module that +enable more advanced and granular controls with respect to what the state is +capable of. + +An example of this along with the new debconf module for installing ldap +client packages on Debian: + +.. code-block:: yaml + + ldap-client-packages: + pkg: + - debconf: salt://debconf/ldap-client.ans + - installed + - names: + - nslcd + - libpam-ldapd + - libnss-ldapd Keyword Arguments and the CLI ------------------ @@ -101,12 +142,6 @@ be passed in the form of ``kwarg=argument``. Matcher Refinements and Changes ------------------------------- -Support has been added for matching minions with Yahoo's range library. This -is handled by passing range syntax with *-R* or *--range* arguments to salt. - -More information at: -https://github.com/grierj/range/wiki/Introduction-to-Range-with-YAML-files - A number of fixes and changes have been applied to the Matcher system. The most noteworthy is the change in the grain matcher. The grain matcher used a regular expression to match the passed data to a grain, but now defaults @@ -129,6 +164,12 @@ And the associated compound matcher suitable for ``top.sls`` is *P*: **NOTE**: The default grains matcher has changed from pcre to glob. This is a backwards incompatible change. +Support has been added for matching minions with Yahoo's range library. This +is handled by passing range syntax with *-R* or *--range* arguments to salt. + +More information at: +https://github.com/grierj/range/wiki/Introduction-to-Range-with-YAML-files + Providers --------- @@ -244,6 +285,15 @@ The goal is that states can be compiled on both the master and the minion allowing for compilation to be split between master and minion. +New File Client +--------------- + +The file client code has been refactored for allowing ``salt://`` uris to +get data from the local server. This will eventually allow for running +salt's configuration management via ``salt-call state.highstate`` to run +without a salt-minion daemon running. + + Solaris Support -------------------- From 6baa783bc18f434ebfea9e9721fc92a244ec8394 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 17:55:58 -0700 Subject: [PATCH 459/598] Add note that git and hg modules have been added --- doc/topics/releases/0.9.8.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 1f3f53838f62..44b24e40e798 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -109,7 +109,7 @@ The *salt-key* command gained the *-D* and *--delete-all* arguments for removing all keys. Be careful with this one! Keyword Arguments and States ------------------ +---------------------------- State modules now accept the ``**kwargs`` argument. This results in all data in a sls file assigned to a state will be made available to the state function. @@ -134,7 +134,7 @@ client packages on Debian: - libnss-ldapd Keyword Arguments and the CLI ------------------- +----------------------------- In the past it was required that all arguments be passed in the proper order to the *salt* and *salt-call* commands. As of 0.9.8, keyword arguments can be @@ -272,6 +272,10 @@ The :doc:`cmd </ref/modules/all/salt.modules.cmdmod>` module and state gained a *shell* keyword argument for specifying a shell other than ``/bin/sh`` on Linux / Unix systems. +New :doc:`git </ref/modules/all/salt.modules.git>` and +:doc:`mercurial </ref/modules/all/salt.modules.hg>` modules have been added +for fans of distributed version control. + In Progress Development ======================= From 2fe1aa57ee133c29dc4761b251bd8a1dcae44252 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 17:57:56 -0700 Subject: [PATCH 460/598] Add zypper to the virtual pkg module list --- doc/ref/modules/all/salt.modules.pkg.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/ref/modules/all/salt.modules.pkg.rst b/doc/ref/modules/all/salt.modules.pkg.rst index 2b7e40dfcc2e..152e9198b892 100644 --- a/doc/ref/modules/all/salt.modules.pkg.rst +++ b/doc/ref/modules/all/salt.modules.pkg.rst @@ -13,3 +13,4 @@ salt.modules.pkg * :mod:`salt.modules.pacman` * :mod:`salt.modules.yumpkg` * :mod:`salt.modules.yumpkg5` +* :mod:`salt.modules.zypper` From dadf6b503af047bf385fd232cb6f50cf04f2d1f8 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 17 Mar 2012 18:03:54 -0700 Subject: [PATCH 461/598] Take the ceilling of batch size percentages before converting to int, #943 --- salt/cli/batch.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/cli/batch.py b/salt/cli/batch.py index b8f69dede7b9..c8ccbe4328d5 100644 --- a/salt/cli/batch.py +++ b/salt/cli/batch.py @@ -2,6 +2,7 @@ Execute batch runs ''' # Import Python libs +import math import time import copy @@ -58,9 +59,11 @@ def get_bnum(self): snum = len(self.minions) try: if self.opts['batch'].startswith('%'): - return int(float(self.opts['batch'][1:]) / 100.0 * snum) + return int(math.ceil( + float(self.opts['batch'][1:]) / 100.0 * snum)) elif self.opts['batch'].endswith('%'): - return int(float(self.opts['batch'][:-1]) / 100.0 * snum) + return int(math.ceil( + float(self.opts['batch'][:-1]) / 100.0 * snum)) except ValueError: print ('Invalid batch data sent: {0}\nData must be in the form' 'of %10, 10% or 3').format(self.opts['batch']) From f834459c4159f4da492b101aa8b9967875785bc3 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 18:05:08 -0700 Subject: [PATCH 462/598] Add rst doc files for a bunch of modules --- doc/ref/modules/all/salt.modules.debconf.rst | 6 ++++++ doc/ref/modules/all/salt.modules.debconfmod.rst | 6 ++++++ doc/ref/modules/all/salt.modules.freebsdservice.rst | 6 ++++++ doc/ref/modules/all/salt.modules.git.rst | 6 ++++++ doc/ref/modules/all/salt.modules.hg.rst | 6 ++++++ doc/ref/modules/all/salt.modules.kvm_hyper.rst | 6 ++++++ doc/ref/modules/all/salt.modules.nginx.rst | 6 ++++++ doc/ref/modules/all/salt.modules.pillar.rst | 6 ++++++ doc/ref/modules/all/salt.modules.zypper.rst | 6 ++++++ 9 files changed, 54 insertions(+) create mode 100644 doc/ref/modules/all/salt.modules.debconf.rst create mode 100644 doc/ref/modules/all/salt.modules.debconfmod.rst create mode 100644 doc/ref/modules/all/salt.modules.freebsdservice.rst create mode 100644 doc/ref/modules/all/salt.modules.git.rst create mode 100644 doc/ref/modules/all/salt.modules.hg.rst create mode 100644 doc/ref/modules/all/salt.modules.kvm_hyper.rst create mode 100644 doc/ref/modules/all/salt.modules.nginx.rst create mode 100644 doc/ref/modules/all/salt.modules.pillar.rst create mode 100644 doc/ref/modules/all/salt.modules.zypper.rst diff --git a/doc/ref/modules/all/salt.modules.debconf.rst b/doc/ref/modules/all/salt.modules.debconf.rst new file mode 100644 index 000000000000..b05ffe9a68bb --- /dev/null +++ b/doc/ref/modules/all/salt.modules.debconf.rst @@ -0,0 +1,6 @@ +==================== +salt.modules.debconf +==================== + +.. automodule:: salt.modules.debconf + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.debconfmod.rst b/doc/ref/modules/all/salt.modules.debconfmod.rst new file mode 100644 index 000000000000..86c81c362beb --- /dev/null +++ b/doc/ref/modules/all/salt.modules.debconfmod.rst @@ -0,0 +1,6 @@ +======================= +salt.modules.debconfmod +======================= + +.. automodule:: salt.modules.debconfmod + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.freebsdservice.rst b/doc/ref/modules/all/salt.modules.freebsdservice.rst new file mode 100644 index 000000000000..1377f063fa6f --- /dev/null +++ b/doc/ref/modules/all/salt.modules.freebsdservice.rst @@ -0,0 +1,6 @@ +=========================== +salt.modules.freebsdservice +=========================== + +.. automodule:: salt.modules.freebsdservice + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.git.rst b/doc/ref/modules/all/salt.modules.git.rst new file mode 100644 index 000000000000..298c22c8a111 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.git.rst @@ -0,0 +1,6 @@ +================ +salt.modules.git +================ + +.. automodule:: salt.modules.git + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.hg.rst b/doc/ref/modules/all/salt.modules.hg.rst new file mode 100644 index 000000000000..34e112587250 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.hg.rst @@ -0,0 +1,6 @@ +=============== +salt.modules.hg +=============== + +.. automodule:: salt.modules.hg + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.kvm_hyper.rst b/doc/ref/modules/all/salt.modules.kvm_hyper.rst new file mode 100644 index 000000000000..e54e2e205aeb --- /dev/null +++ b/doc/ref/modules/all/salt.modules.kvm_hyper.rst @@ -0,0 +1,6 @@ +================ +salt.modules.kvm_hyper +================ + +.. automodule:: salt.modules.kvm_hyper + :members: diff --git a/doc/ref/modules/all/salt.modules.nginx.rst b/doc/ref/modules/all/salt.modules.nginx.rst new file mode 100644 index 000000000000..409073502a91 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.nginx.rst @@ -0,0 +1,6 @@ +================== +salt.modules.nginx +================== + +.. automodule:: salt.modules.nginx + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.pillar.rst b/doc/ref/modules/all/salt.modules.pillar.rst new file mode 100644 index 000000000000..e24b307145a6 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.pillar.rst @@ -0,0 +1,6 @@ +=================== +salt.modules.pillar +=================== + +.. automodule:: salt.modules.pillar + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.zypper.rst b/doc/ref/modules/all/salt.modules.zypper.rst new file mode 100644 index 000000000000..f8c25960c435 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.zypper.rst @@ -0,0 +1,6 @@ +=================== +salt.modules.zypper +=================== + +.. automodule:: salt.modules.zypper + :members: \ No newline at end of file From fabe5ba4710afcca2bbc0f2593368abeff237aaf Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 18:05:37 -0700 Subject: [PATCH 463/598] Add docs for the new ruby states and update the cmd one --- doc/ref/modules/all/salt.modules.cmdmod.rst | 4 ++-- doc/ref/states/all/salt.states.gem.rst | 6 ++++++ doc/ref/states/all/salt.states.rvm.rst | 6 ++++++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 doc/ref/states/all/salt.states.gem.rst create mode 100644 doc/ref/states/all/salt.states.rvm.rst diff --git a/doc/ref/modules/all/salt.modules.cmdmod.rst b/doc/ref/modules/all/salt.modules.cmdmod.rst index 6451b7e59dd0..62f8fad6aabe 100644 --- a/doc/ref/modules/all/salt.modules.cmdmod.rst +++ b/doc/ref/modules/all/salt.modules.cmdmod.rst @@ -2,5 +2,5 @@ salt.modules.cmd ================ -.. automodule:: salt.modules.cmd - :members: \ No newline at end of file +.. automodule:: salt.modules.cmdmod + :members: diff --git a/doc/ref/states/all/salt.states.gem.rst b/doc/ref/states/all/salt.states.gem.rst new file mode 100644 index 000000000000..2d9a52bc3018 --- /dev/null +++ b/doc/ref/states/all/salt.states.gem.rst @@ -0,0 +1,6 @@ +=============== +salt.states.gem +=============== + +.. automodule:: salt.states.gem + :members: \ No newline at end of file diff --git a/doc/ref/states/all/salt.states.rvm.rst b/doc/ref/states/all/salt.states.rvm.rst new file mode 100644 index 000000000000..5d6ee4f8bd32 --- /dev/null +++ b/doc/ref/states/all/salt.states.rvm.rst @@ -0,0 +1,6 @@ +=============== +salt.states.rvm +=============== + +.. automodule:: salt.states.rvm + :members: \ No newline at end of file From 277e7c0ce8d780484b8a01e8b1a99f7fbed8c955 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sat, 17 Mar 2012 18:16:11 -0700 Subject: [PATCH 464/598] Only check ceiling on batches < 1, #943 --- salt/cli/batch.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/salt/cli/batch.py b/salt/cli/batch.py index c8ccbe4328d5..16f77b6b6a91 100644 --- a/salt/cli/batch.py +++ b/salt/cli/batch.py @@ -44,7 +44,7 @@ def __gather_minions(self): args.append('compound') else: args.append('glob') - + fret = [] for ret in self.local.cmd_iter(*args): for minion in ret: @@ -56,18 +56,19 @@ def get_bnum(self): ''' Return the active number of minions to maintain ''' - snum = len(self.minions) + partition = lambda x: float(x) / 100.0 * len(self.minions) try: - if self.opts['batch'].startswith('%'): - return int(math.ceil( - float(self.opts['batch'][1:]) / 100.0 * snum)) - elif self.opts['batch'].endswith('%'): - return int(math.ceil( - float(self.opts['batch'][:-1]) / 100.0 * snum)) + if '%' in self.opts['batch']: + res = partition(float(self.opts['batch'].strip('%'))) + if res < 1: + return int(math.ceil(res)) + else: + return int(res) + else: + return int(self.opts['batch']) except ValueError: print ('Invalid batch data sent: {0}\nData must be in the form' 'of %10, 10% or 3').format(self.opts['batch']) - return int(self.opts['batch']) def run(self): ''' From 066ab17c79696d17916c50aecccff09fb7e07bd6 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 18:43:53 -0700 Subject: [PATCH 465/598] Make the release 0.9.8.pre --- salt/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/version.py b/salt/version.py index b25cb3006a87..d044332b8eaa 100644 --- a/salt/version.py +++ b/salt/version.py @@ -1,2 +1,2 @@ -__version_info__ = (0, 9, 7) +__version_info__ = (0, 9, 8, 'pre') __version__ = '.'.join(map(str, __version_info__)) From 93be588ebb7bd9b24ee096a2116c6e0361a68796 Mon Sep 17 00:00:00 2001 From: David Bishop <david@gnuconsulting.com> Date: Sat, 17 Mar 2012 21:50:16 -0400 Subject: [PATCH 466/598] add sentence pointing to _modules dir for custom modules --- doc/ref/modules/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/ref/modules/index.rst b/doc/ref/modules/index.rst index 656cd2478b2b..6e14849c6760 100644 --- a/doc/ref/modules/index.rst +++ b/doc/ref/modules/index.rst @@ -12,7 +12,8 @@ Easy Modules to write ===================== Salt modules are amazingly simple to write, just write a regular Python module -or a regular Cython module and place it in the ``salt/modules`` directory. +or a regular Cython module and place it in the ``salt/modules`` directory. You +can also place them in a directory called ``_modules/`` in your state directory. Since Salt modules are just Python/Cython modules there are no restraints as to what you can put inside of a salt module, and if a Salt module has errors and From 6e220a41e9f1baa72d838e35ec63246f98985ea0 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 18:43:07 -0700 Subject: [PATCH 467/598] Adding link to Pillar documentation --- doc/topics/releases/0.9.8.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 44b24e40e798..1a4197b762b6 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -82,10 +82,11 @@ https://launchpad.net/~saltstack/+archive/salt Major Features ============== + Pillar ------ -Pillar offers an interface to declare variable data on the master that is then +:doc:`Pillar </topics/pillar/index>` offers an interface to declare variable data on the master that is then assigned to the minions. The pillar data is made available to all modules, states, sls files etc. It is compiled on the master and is declared using the existing renderer system. This means that learning pillar should be fairly From 94e7e7e58baeee54ab422c4a98427a51f307817f Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 19:38:13 -0700 Subject: [PATCH 468/598] Update the docstrings for file.directory and file.recurse This documents the newly added arguments --- salt/states/file.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/salt/states/file.py b/salt/states/file.py index 21c18a87de31..948e9acb35d7 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -825,6 +825,10 @@ def directory(name, Make sure that only files that are set up by salt and required by this function are kept. If this option is set then everything in this directory will be deleted unless it is required. + + require + Require other resources such as packages or files + ''' mode = __manage_mode(mode) ret = {'name': name, @@ -977,6 +981,23 @@ def recurse(name, Make sure that only files that are set up by salt and required by this function are kept. If this option is set then everything in this directory will be deleted unless it is required. + + require + Require other resources such as packages or files + + user + The user to own the directory, this defaults to the user salt is + running as on the minion + + group + The group ownership set for the directory, this defaults to the group + salt is running as on the minion + + dir_mode + The permissions mode to set any directories created + + file_mode + The permissions mode to set any files created ''' ret = {'name': name, 'changes': {}, From 8c75013112d4b5216204fb263b4830fd16b15b8f Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 04:31:10 +0000 Subject: [PATCH 469/598] reverting true to false for no_site_packages --- salt/modules/virtualenv.py | 4 ++-- salt/states/virtualenv.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/virtualenv.py b/salt/modules/virtualenv.py index 690f1f752106..56ec629a642d 100644 --- a/salt/modules/virtualenv.py +++ b/salt/modules/virtualenv.py @@ -11,7 +11,7 @@ def create(path, venv_bin=__opts__['venv_bin'], - no_site_packages=True, + no_site_packages=False, system_site_packages=False, clear=False, python='', @@ -27,7 +27,7 @@ def create(path, venv_bin : 'virtualenv' The name (and optionally path) of the virtualenv command. This can also be set globally in the minion config file as ``virtualenv.venv_bin``. - no_site_packages : True + no_site_packages : False Passthrough argument given to virtualenv system_site_packages : False Passthrough argument given to virtualenv diff --git a/salt/states/virtualenv.py b/salt/states/virtualenv.py index 8531200a7176..a0ba019a7524 100644 --- a/salt/states/virtualenv.py +++ b/salt/states/virtualenv.py @@ -10,7 +10,7 @@ def manage(name, venv_bin='virtualenv', requirements='', - no_site_packages=True, + no_site_packages=False, system_site_packages=False, clear=False, python='', From 8aef521ac445cac82abff7673baba3129acb34a3 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 05:02:43 +0000 Subject: [PATCH 470/598] adding the ability to use the distribute flag --- salt/modules/virtualenv.py | 4 ++++ salt/states/virtualenv.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/salt/modules/virtualenv.py b/salt/modules/virtualenv.py index 56ec629a642d..e3d2fcd5e522 100644 --- a/salt/modules/virtualenv.py +++ b/salt/modules/virtualenv.py @@ -13,6 +13,7 @@ def create(path, venv_bin=__opts__['venv_bin'], no_site_packages=False, system_site_packages=False, + distribute=False, clear=False, python='', extra_search_dir='', @@ -31,6 +32,8 @@ def create(path, Passthrough argument given to virtualenv system_site_packages : False Passthrough argument given to virtualenv + distribute : False + Passthrough argument given to virtualenv clear : False Passthrough argument given to virtualenv python : (default) @@ -57,6 +60,7 @@ def create(path, args=''.join([ ' --no-site-packages' if no_site_packages else '', ' --system-site-packages' if system_site_packages else '', + ' --distribute' if distribute else '', ' --clear' if clear else '', ' --python {0}'.format(python) if python else '', ' --extra-search-dir {0}'.format(extra_search_dir diff --git a/salt/states/virtualenv.py b/salt/states/virtualenv.py index a0ba019a7524..2dce7052b282 100644 --- a/salt/states/virtualenv.py +++ b/salt/states/virtualenv.py @@ -12,6 +12,7 @@ def manage(name, requirements='', no_site_packages=False, system_site_packages=False, + distribute=False, clear=False, python='', extra_search_dir='', @@ -72,6 +73,7 @@ def manage(name, venv_bin=venv_bin, no_site_packages=no_site_packages, system_site_packages=system_site_packages, + distribute=distribute, clear=clear, python=python, extra_search_dir=extra_search_dir, From 141050666d73baa7c7b7582b4795b8dfcdc75d0a Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 05:13:02 +0000 Subject: [PATCH 471/598] typo, need space --- salt/modules/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index 22248c6a8f47..a077f342cf4b 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -151,7 +151,7 @@ def install(pkgs=None, cmd=cmd, pkg=pkg) if requirements: - cmd = '{cmd} --requirements{requirements} '.format( + cmd = '{cmd} --requirements {requirements} '.format( cmd=cmd, requirements=requirements) if log: From 1572662123d76d08f04294a70538516f74c90c6a Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 22:52:26 -0700 Subject: [PATCH 472/598] Add example of how to call modules with kwargs --- doc/topics/releases/0.9.8.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 1a4197b762b6..d5e9e9d9a261 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -141,6 +141,11 @@ In the past it was required that all arguments be passed in the proper order to the *salt* and *salt-call* commands. As of 0.9.8, keyword arguments can be passed in the form of ``kwarg=argument``. +.. code-block:: sh + + # salt -G 'type:dev' git.clone repository=https://github.com/saltstack/salt.git cwd=/tmp/salt + + Matcher Refinements and Changes ------------------------------- From 6bd3909e2bee269e8dc01a6e18ee90ba89174c5d Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 06:04:06 +0000 Subject: [PATCH 473/598] flag is requirement not requirements --- salt/modules/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index a077f342cf4b..00bce3cd33df 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -151,7 +151,7 @@ def install(pkgs=None, cmd=cmd, pkg=pkg) if requirements: - cmd = '{cmd} --requirements {requirements} '.format( + cmd = '{cmd} --requirement {requirements} '.format( cmd=cmd, requirements=requirements) if log: From 856f8edf333b529d5c288c8a73969ee2c5aaf622 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 23:10:34 -0700 Subject: [PATCH 474/598] Add quoting around requirements in the pip module and fix another tyop --- salt/modules/pip.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index 00bce3cd33df..78fc5ac37b9c 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -151,7 +151,7 @@ def install(pkgs=None, cmd=cmd, pkg=pkg) if requirements: - cmd = '{cmd} --requirement {requirements} '.format( + cmd = '{cmd} --requirement "{requirements}" '.format( cmd=cmd, requirements=requirements) if log: @@ -190,13 +190,13 @@ def install(pkgs=None, if index_url: if not index_url.startswith("http://"): raise Exception("'%s' must be a valid url" % index_url) - cmd = '{cmd} --index_url={index_url} '.format( + cmd = '{cmd} --index_url="{index_url}" '.format( cmd=cmd, index_url=index_url) if extra_index_url: if not extra_index_url.startswith("http://"): raise Exception("'%s' must be a valid url" % extra_index_url) - cmd = '{cmd} --extra_index_url={extra_index_url} '.format( + cmd = '{cmd} --extra_index_url="{extra_index_url}" '.format( cmd=cmd, extra_index_url=extra_index_url) if no_index: @@ -307,7 +307,7 @@ def uninstall(pkgs=None, cmd=cmd, pkg=pkg) if requirements: - cmd = '{cmd} --requirements{requirements} '.format( + cmd = '{cmd} --requirements "{requirements}" '.format( cmd=cmd, requirements=requirements) if log: From a92a0317745a6c268938f2d2a09a6efbf428d094 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 23:30:07 -0700 Subject: [PATCH 475/598] Add "user" kwarg to function in the git module This is for #951 --- salt/modules/git.py | 65 ++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 949f4fdbc1e4..3f69919a2186 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -1,15 +1,16 @@ ''' Support for the Git SCM ''' + import os -def _git_getdir(cwd): +def _git_getdir(cwd, user=None): ''' Returns the absolute path to the top-level of a given repo because some Git commands are sensitive to where they're run from (archive for one) ''' cmd_bare = 'git rev-parse --is-bare-repository' - is_bare = __salt__['cmd.run_stdout'](cmd_bare, cwd) == 'true' + is_bare = __salt__['cmd.run_stdout'](cmd_bare, cwd, runas=user) == 'true' if is_bare: return cwd @@ -17,7 +18,7 @@ def _git_getdir(cwd): cmd_toplvl = 'git rev-parse --show-toplevel' return __salt__['cmd.run'](cmd_toplvl, cwd) -def revision(cwd, rev='HEAD', short=False): +def revision(cwd, rev='HEAD', short=False, user=None): ''' Returns the long hash of a given identifier (hash, branch, tag, HEAD, etc) @@ -26,14 +27,14 @@ def revision(cwd, rev='HEAD', short=False): salt '*' git.revision /path/to/repo mybranch ''' cmd = 'git rev-parse {0}{1}'.format('--short ' if short else '', rev) - result = __salt__['cmd.run_all'](cmd, cwd) + result = __salt__['cmd.run_all'](cmd, cwd, runas=user) if result['retcode'] == 0: - return result['stdout'].strip('\n') + return result['stdout'] else: return '' -def clone(cwd, repository, opts=''): +def clone(cwd, repository, opts=None, user=None): ''' Clone a new repository @@ -45,18 +46,21 @@ def clone(cwd, repository, opts=''): git://github.com/saltstack/salt.git '--bare --origin github' ''' + if not opts: + opts = '' cmd = 'git clone {0} {1} {2}'.format(repository, cwd, opts) - return __salt__['cmd.run'](cmd) -def describe(cwd, rev='HEAD'): + return __salt__['cmd.run'](cmd, runas=user) + +def describe(cwd, rev='HEAD', user=None): ''' Returns the git describe string (or the SHA hash if there are no tags) for the given revision ''' cmd = 'git describe {0}'.format(rev) - return __salt__['cmd.run_stdout'](cmd, cwd=cwd).strip('\n') + return __salt__['cmd.run_stdout'](cmd, cwd=cwd, runas=user) -def archive(cwd, output, rev='HEAD', fmt='', prefix=''): +def archive(cwd, output, rev='HEAD', fmt='', prefix='', user=None): ''' Export a tarball from the repository @@ -67,7 +71,7 @@ def archive(cwd, output, rev='HEAD', fmt='', prefix=''): salt '*' git.archive /path/to/repo /path/to/archive.tar.gz ''' - basename = '{0}/'.format(os.path.basename(_git_getdir(cwd)).strip('\n')) + basename = '{0}/'.format(os.path.basename(_git_getdir(cwd, user=user))) cmd = 'git archive{prefix}{fmt} -o {output} {rev}'.format( rev = rev, @@ -75,9 +79,9 @@ def archive(cwd, output, rev='HEAD', fmt='', prefix=''): fmt = ' --format={0}'.format(fmt) if fmt else '', prefix = ' --prefix="{0}"'.format(prefix if prefix else basename)) - return __salt__['cmd.run'](cmd, cwd=cwd) + return __salt__['cmd.run'](cmd, cwd=cwd, runas=user) -def fetch(cwd, opts=''): +def fetch(cwd, opts=None, user=None): ''' Perform a fetch on the given repository @@ -85,9 +89,13 @@ def fetch(cwd, opts=''): salt '*' git.fetch /path/to/repo '--all' ''' - return __salt__['cmd.run']('git fetch {0}'.format(opts), cwd=cwd) + if not opts: + opts = '' + cmd = 'git fetch {0}'.format(opts) -def pull(cwd, opts=''): + return __salt__['cmd.run'](cmd, cwd=cwd, runas=user) + +def pull(cwd, opts=None, user=None): ''' Perform a pull on the given repository @@ -95,24 +103,28 @@ def pull(cwd, opts=''): salt '*' git.pull /path/to/repo '--rebase origin master' ''' - return __salt__['cmd.run']('git pull {0}'.format(opts), cwd=cwd) + if not opts: + opts = '' + return __salt__['cmd.run']('git pull {0}'.format(opts), cwd=cwd, runas=user) -def checkout(cwd, rev, force=False, opts=''): +def checkout(cwd, rev, force=False, opts=None, user=None): ''' Checkout a given revision Usage:: - salt '*' git.checkout /path/to/repo somebranch + salt '*' git.checkout /path/to/repo somebranch user=jeff - salt '*' git.checkout /path/to/repo 'testbranch -- conf/file1 file2' + salt '*' git.checkout /path/to/repo opts='testbranch -- conf/file1 file2' salt '*' git.checkout /path/to/repo 'origin/mybranch --track' ''' - cmd = 'git checkout{0} {1} {2}'.format(' -f' if force else '', rev, opts) - return __salt__['cmd.run'](cmd, cwd=cwd) + if not opts: + opts = '' + cmd = 'git checkout {0} {1} {2}'.format(' -f' if force else '', rev, opts) + return __salt__['cmd.run'](cmd, cwd=cwd, runas=user) -def merge(cwd, branch='@{upstream}', opts=''): +def merge(cwd, branch='@{upstream}', opts=None, user=None): ''' Merge a given branch @@ -128,12 +140,15 @@ def merge(cwd, branch='@{upstream}', opts=''): salt '*' git.fetch /path/to/repo salt '*' git.merge /path/to/repo @{upstream} ''' + if not opts: + opts = '' cmd = 'git merge {0}{1} {2}'.format( branch, opts) - return __salt__['cmd.run'](cmd, cwd).strip('\n') -def init(cwd, opts=''): + return __salt__['cmd.run'](cmd, cwd, runas=user) + +def init(cwd, opts=None, user=None): ''' Init a new repository @@ -142,4 +157,4 @@ def init(cwd, opts=''): salt '*' git.init /path/to/repo.git '--bare' ''' cmd = 'git init {0} {1}'.format(cwd, opts) - return __salt__['cmd.run'](cmd).strip('\n') + return __salt__['cmd.run'](cmd, runas=user) From 5fbd73d4dd5b5215cc86afa81fc09fc7bf3593b5 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sat, 17 Mar 2012 23:36:46 -0700 Subject: [PATCH 476/598] Add user flag for git to the release notes --- doc/topics/releases/0.9.8.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index d5e9e9d9a261..07cc73008253 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -143,7 +143,8 @@ passed in the form of ``kwarg=argument``. .. code-block:: sh - # salt -G 'type:dev' git.clone repository=https://github.com/saltstack/salt.git cwd=/tmp/salt + # salt -G 'type:dev' git.clone \ + repository=https://github.com/saltstack/salt.git cwd=/tmp/salt user=jeff Matcher Refinements and Changes From 42bdf32815a3969433a64a275a0dea1705f809ea Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 06:39:14 +0000 Subject: [PATCH 477/598] adding state for postgres db's --- salt/states/postgres_database.py | 61 ++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 salt/states/postgres_database.py diff --git a/salt/states/postgres_database.py b/salt/states/postgres_database.py new file mode 100644 index 000000000000..9e9bfb73b83b --- /dev/null +++ b/salt/states/postgres_database.py @@ -0,0 +1,61 @@ +''' +Postgres Database Management +========================= +The postgres_database module is used to create and manage Postgres databases, databases can be set +as either absent or present + +.. code-block:: yaml + + frank: + postgres_database: + - present +''' + +def present(name): + ''' + Ensure that the named database is present with the specified properties + + name + The name of the database to manage + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': 'Database {0} is already present'.format(name)} + # check if database exists + if __salt__['postgres.db_exists'](name): + return ret + + # The database is not present, make it! + if __salt__['postgres.db_create'](name): + ret['comment'] = 'The database {0} has been created'.format(name) + ret['changes'][name] = 'Present' + else: + ret['comment'] = 'Failed to create database {0}'.format(name) + ret['result'] = False + + return ret + + +def absent(name): + ''' + Ensure that the named database is absent + + name + The name of the database to remove + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + #check if db exists and remove it + if __salt__['postgres.db_exists'](name): + if __salt__['postgres.db_remove'](name): + ret['comment'] = 'Database {0} has been removed'.format(name) + ret['changes'][name] = 'Absent' + return ret + + # fallback + ret['comment'] = 'Database {0} is not present, so it cannot be removed'.format(name) + return ret From a47d12035e7432ffee44eb3ee00abb450fefe5fb Mon Sep 17 00:00:00 2001 From: Nick Lang <nick@nicklang.com> Date: Sun, 18 Mar 2012 13:24:19 -0600 Subject: [PATCH 478/598] adding initla postgres module --- salt/modules/postgres.py | 384 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 384 insertions(+) create mode 100644 salt/modules/postgres.py diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py new file mode 100644 index 000000000000..fb1a45793718 --- /dev/null +++ b/salt/modules/postgres.py @@ -0,0 +1,384 @@ +''' +Module to provide MySQL compatibility to salt. + +In order to connect to MySQL, certain configuration is required +in /etc/salt/minion on the relevant minions. Some sample configs +might look like:: + + postgres.host: 'localhost' + postgres.port: 3306 + postgres.user: 'root' + postgres.pass: '' + postgres.db: 'mysql' + +You can also use a defaults file:: + + mysql.default_file: '/etc/mysql/debian.cnf' + +Required python modules: MySQLdb +''' + +import logging + +log = logging.getLogger(__name__) +__opts__ = {} + + +def version(user, password): + ''' + Return the version of a MySQL server using the output + from the ``SELECT VERSION()`` query. + + CLI Example:: + + salt '*' postgres.version + ''' + __salt__['cmd.run']('psql --version') + + +''' +Database related actions +''' + + +def db_list(user, host): + ''' + Return a list of databases of a MySQL server using the output + from the ``SHOW DATABASES`` query. + + CLI Example:: + + salt '*' postgres.db_list + ''' + cmd = "psql -l -U {user} -h {host}".format( + user=user, host=host) + + __salt__['cmd.run'](cmd) + + +def db_exists(user, host, name): + ''' + Checks if a database exists on the MySQL server. + + CLI Example:: + + salt '*' mysql.db_exists 'dbname' + ''' + databases = __salt__['postgres.db_list'](user, host) + return name in databases + + +def db_create(user, host, name, **kwargs): + ''' + Adds a databases to the MySQL server. + + CLI Example:: + + salt '*' mysql.db_create 'dbname' + ''' + # check if db exists + if db_exists(name): + log.info("DB '{0}' already exists".format(name,)) + return False + + cmd = 'create_db {name} '.format(name) + for param, value in kwargs.iteritems(): + cmd = '{cmd} {param} {value} '.format( + cmd=cmd, param=param, value=value) + + __salt__['cmd.run'](cmd) + + +def db_remove(name): + ''' + Removes a databases from the MySQL server. + + CLI Example:: + + salt '*' mysql.db_remove 'dbname' + ''' + # check if db exists + if not db_exists(name): + log.info("DB '{0}' does not exist".format(name,)) + return False + + # db doesnt exist, proceed + cmd = 'dropdb {name} '.format(name) + __salt__['cmd.run'](cmd) + + +''' +User related actions +''' +def user_create(user, + host='localhost', + password=None, + password_hash=None): + ''' + Creates a MySQL user. + + CLI Examples:: + + salt '*' mysql.user_create 'username' 'hostname' 'password + + salt '*' mysql.user_create 'username' 'hostname' password_hash='hash' + ''' + if user_exists(user,host): + log.info("User '{0}'@'{1}' already exists".format(user,host,)) + return False + + db = connect() + cur = db.cursor () + query = "CREATE USER '%s'@'%s'" % (user, host,) + if password is not None: + query = query + " IDENTIFIED BY '%s'" % password + elif password_hash is not None: + query = query + " IDENTIFIED BY PASSWORD '%s'" % password_hash + + log.debug("Query: {0}".format(query,)) + cur.execute( query ) + + if user_exists(user,host): + log.info("User '{0}'@'{1}' has been created".format(user,host,)) + return True + + log.info("User '{0}'@'{1}' is not created".format(user,host,)) + return False + +def user_chpass(user, + host='localhost', + password=None, + password_hash=None): + ''' + Change password for MySQL user + + CLI Examples:: + + salt '*' mysql.user_chpass frank localhost newpassword + + salt '*' mysql.user_chpass frank localhost password_hash='hash' + ''' + if password is None or password_hash is None: + log.error('No password provided') + return False + elif password is not None: + password_sql = "PASSWORD(\"%s\")" % password + elif password_hash is not None: + password_sql = "\"%s\"" % password_hash + + db = connect() + cur = db.cursor () + query = "UPDATE mysql.user SET password=%s WHERE User='%s' AND Host = '%s';" % (password_sql,user,host,) + log.debug("Query: {0}".format(query,)) + if cur.execute( query ): + log.info("Password for user '{0}'@'{1}' has been changed".format(user,host,)) + return True + + log.info("Password for user '{0}'@'{1}' is not changed".format(user,host,)) + return False + +def user_remove(user, + host='localhost'): + ''' + Delete MySQL user + + CLI Example:: + + salt '*' mysql.user_remove frank localhost + ''' + db = connect() + cur = db.cursor () + query = "DROP USER '%s'@'%s'" % (user, host,) + log.debug("Query: {0}".format(query,)) + cur.execute(query) + result = cur.fetchone() + if not user_exists(user,host): + log.info("User '{0}'@'{1}' has been removed".format(user,host,)) + return True + + log.info("User '{0}'@'{1}' has NOT been removed".format(user,host,)) + return False + +''' +Maintenance +''' +def db_check(name, + table=None): + ''' + Repairs the full database or just a given table + + CLI Example:: + + salt '*' mysql.db_check dbname + ''' + ret = [] + if table is None: + # we need to check all tables + tables = db_tables(name) + for table in tables: + log.info("Checking table '%s' in db '%s..'".format(name,table,)) + ret.append( __check_table(name,table) ) + else: + log.info("Checking table '%s' in db '%s'..".format(name,table,)) + ret = __check_table(name,table) + return ret + +def db_repair(name, + table=None): + ''' + Repairs the full database or just a given table + + CLI Example:: + + salt '*' mysql.db_repair dbname + ''' + ret = [] + if table is None: + # we need to repair all tables + tables = db_tables(name) + for table in tables: + log.info("Repairing table '%s' in db '%s..'".format(name,table,)) + ret.append( __repair_table(name,table) ) + else: + log.info("Repairing table '%s' in db '%s'..".format(name,table,)) + ret = __repair_table(name,table) + return ret + +def db_optimize(name, + table=None): + ''' + Optimizes the full database or just a given table + + CLI Example:: + + salt '*' mysql.db_optimize dbname + ''' + ret = [] + if table is None: + # we need to optimize all tables + tables = db_tables(name) + for table in tables: + log.info("Optimizing table '%s' in db '%s..'".format(name,table,)) + ret.append( __optimize_table(name,table) ) + else: + log.info("Optimizing table '%s' in db '%s'..".format(name,table,)) + ret = __optimize_table(name,table) + return ret + +''' +Grants +''' +def __grant_generate(grant, + database, + user, + host='localhost', + grant_option=False, + escape=True): + # todo: Re-order the grant so it is according to the SHOW GRANTS for xxx@yyy query (SELECT comes first, etc) + grant = grant.replace(',', ', ').upper() + + db_part = database.rpartition('.') + db = db_part[0] + table = db_part[2] + + if escape: + db = "`%s`" % db + table = "`%s`" % table + query = "GRANT %s ON %s.%s TO '%s'@'%s'" % (grant, db, table, user, host,) + if grant_option: + query += " WITH GRANT OPTION" + log.debug("Query generated: {0}".format(query,)) + return query + +def user_grants(user, + host='localhost'): + ''' + Shows the grants for the given MySQL user (if it exists) + + CLI Example:: + + salt '*' mysql.user_grants 'frank' 'localhost' + ''' + if not user_exists(user): + log.info("User '{0}' does not exist".format(user,)) + return False + + ret = [] + db = connect() + cur = db.cursor() + query = "SHOW GRANTS FOR '%s'@'%s'" % (user,host,) + log.debug("Doing query: {0}".format(query,)) + + cur.execute(query) + results = cur.fetchall() + for grant in results: + ret.append(grant[0]) + log.debug(ret) + return ret + +def grant_exists(grant, + database, + user, + host='localhost', + grant_option=False, + escape=True): + # todo: This function is a bit tricky, since it requires the ordering to be exactly the same. + # perhaps should be replaced/reworked with a better/cleaner solution. + target = __grant_generate(grant, database, user, host, grant_option, escape) + + if target in user_grants(user, host): + log.debug("Grant exists.") + return True + + log.debug("Grant does not exist, or is perhaps not ordered properly?") + return False + +def grant_add(grant, + database, + user, + host='localhost', + grant_option=False, + escape=True): + ''' + Adds a grant to the MySQL server. + + CLI Example:: + + salt '*' mysql.grant_add 'SELECT|INSERT|UPDATE|...' 'database.*' 'frank' 'localhost' + ''' + # todo: validate grant + db = connect() + cur = db.cursor() + + query = __grant_generate(grant, database, user, host, grant_option, escape) + log.debug("Query: {0}".format(query,)) + if cur.execute( query ): + log.info("Grant '{0}' created") + return True + return False + +def grant_revoke(grant, + database, + user, + host='localhost', + grant_option=False): + ''' + Removes a grant from the MySQL server. + + CLI Example:: + + salt '*' mysql.grant_revoke 'SELECT,INSERT,UPDATE' 'database.*' 'frank' 'localhost' + ''' + # todo: validate grant + db = connect() + cur = db.cursor() + + if grant_option: + grant += ", GRANT OPTION" + query = "REVOKE %s ON %s FROM '%s'@'%s';" % (grant, database, user, host,) + log.debug("Query: {0}".format(query,)) + if cur.execute( query ): + log.info("Grant '{0}' revoked") + return True + return False + From 80d23489e2fd505d8a6392a4360467788af4d155 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 20:04:24 +0000 Subject: [PATCH 479/598] making some changes to the postgres module... trying to get version() to work --- salt/modules/postgres.py | 263 +-------------------------------------- 1 file changed, 3 insertions(+), 260 deletions(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index fb1a45793718..72062b4127d5 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -24,7 +24,7 @@ __opts__ = {} -def version(user, password): +def version(): ''' Return the version of a MySQL server using the output from the ``SELECT VERSION()`` query. @@ -33,8 +33,7 @@ def version(user, password): salt '*' postgres.version ''' - __salt__['cmd.run']('psql --version') - + return __salt__['cmd.run']('psql --version') ''' Database related actions @@ -123,262 +122,6 @@ def user_create(user, salt '*' mysql.user_create 'username' 'hostname' password_hash='hash' ''' - if user_exists(user,host): - log.info("User '{0}'@'{1}' already exists".format(user,host,)) - return False - - db = connect() - cur = db.cursor () - query = "CREATE USER '%s'@'%s'" % (user, host,) - if password is not None: - query = query + " IDENTIFIED BY '%s'" % password - elif password_hash is not None: - query = query + " IDENTIFIED BY PASSWORD '%s'" % password_hash - - log.debug("Query: {0}".format(query,)) - cur.execute( query ) - - if user_exists(user,host): - log.info("User '{0}'@'{1}' has been created".format(user,host,)) - return True + pass - log.info("User '{0}'@'{1}' is not created".format(user,host,)) - return False - -def user_chpass(user, - host='localhost', - password=None, - password_hash=None): - ''' - Change password for MySQL user - - CLI Examples:: - - salt '*' mysql.user_chpass frank localhost newpassword - - salt '*' mysql.user_chpass frank localhost password_hash='hash' - ''' - if password is None or password_hash is None: - log.error('No password provided') - return False - elif password is not None: - password_sql = "PASSWORD(\"%s\")" % password - elif password_hash is not None: - password_sql = "\"%s\"" % password_hash - - db = connect() - cur = db.cursor () - query = "UPDATE mysql.user SET password=%s WHERE User='%s' AND Host = '%s';" % (password_sql,user,host,) - log.debug("Query: {0}".format(query,)) - if cur.execute( query ): - log.info("Password for user '{0}'@'{1}' has been changed".format(user,host,)) - return True - - log.info("Password for user '{0}'@'{1}' is not changed".format(user,host,)) - return False - -def user_remove(user, - host='localhost'): - ''' - Delete MySQL user - - CLI Example:: - - salt '*' mysql.user_remove frank localhost - ''' - db = connect() - cur = db.cursor () - query = "DROP USER '%s'@'%s'" % (user, host,) - log.debug("Query: {0}".format(query,)) - cur.execute(query) - result = cur.fetchone() - if not user_exists(user,host): - log.info("User '{0}'@'{1}' has been removed".format(user,host,)) - return True - - log.info("User '{0}'@'{1}' has NOT been removed".format(user,host,)) - return False - -''' -Maintenance -''' -def db_check(name, - table=None): - ''' - Repairs the full database or just a given table - - CLI Example:: - - salt '*' mysql.db_check dbname - ''' - ret = [] - if table is None: - # we need to check all tables - tables = db_tables(name) - for table in tables: - log.info("Checking table '%s' in db '%s..'".format(name,table,)) - ret.append( __check_table(name,table) ) - else: - log.info("Checking table '%s' in db '%s'..".format(name,table,)) - ret = __check_table(name,table) - return ret - -def db_repair(name, - table=None): - ''' - Repairs the full database or just a given table - - CLI Example:: - - salt '*' mysql.db_repair dbname - ''' - ret = [] - if table is None: - # we need to repair all tables - tables = db_tables(name) - for table in tables: - log.info("Repairing table '%s' in db '%s..'".format(name,table,)) - ret.append( __repair_table(name,table) ) - else: - log.info("Repairing table '%s' in db '%s'..".format(name,table,)) - ret = __repair_table(name,table) - return ret - -def db_optimize(name, - table=None): - ''' - Optimizes the full database or just a given table - - CLI Example:: - - salt '*' mysql.db_optimize dbname - ''' - ret = [] - if table is None: - # we need to optimize all tables - tables = db_tables(name) - for table in tables: - log.info("Optimizing table '%s' in db '%s..'".format(name,table,)) - ret.append( __optimize_table(name,table) ) - else: - log.info("Optimizing table '%s' in db '%s'..".format(name,table,)) - ret = __optimize_table(name,table) - return ret - -''' -Grants -''' -def __grant_generate(grant, - database, - user, - host='localhost', - grant_option=False, - escape=True): - # todo: Re-order the grant so it is according to the SHOW GRANTS for xxx@yyy query (SELECT comes first, etc) - grant = grant.replace(',', ', ').upper() - - db_part = database.rpartition('.') - db = db_part[0] - table = db_part[2] - - if escape: - db = "`%s`" % db - table = "`%s`" % table - query = "GRANT %s ON %s.%s TO '%s'@'%s'" % (grant, db, table, user, host,) - if grant_option: - query += " WITH GRANT OPTION" - log.debug("Query generated: {0}".format(query,)) - return query - -def user_grants(user, - host='localhost'): - ''' - Shows the grants for the given MySQL user (if it exists) - - CLI Example:: - - salt '*' mysql.user_grants 'frank' 'localhost' - ''' - if not user_exists(user): - log.info("User '{0}' does not exist".format(user,)) - return False - - ret = [] - db = connect() - cur = db.cursor() - query = "SHOW GRANTS FOR '%s'@'%s'" % (user,host,) - log.debug("Doing query: {0}".format(query,)) - - cur.execute(query) - results = cur.fetchall() - for grant in results: - ret.append(grant[0]) - log.debug(ret) - return ret - -def grant_exists(grant, - database, - user, - host='localhost', - grant_option=False, - escape=True): - # todo: This function is a bit tricky, since it requires the ordering to be exactly the same. - # perhaps should be replaced/reworked with a better/cleaner solution. - target = __grant_generate(grant, database, user, host, grant_option, escape) - - if target in user_grants(user, host): - log.debug("Grant exists.") - return True - - log.debug("Grant does not exist, or is perhaps not ordered properly?") - return False - -def grant_add(grant, - database, - user, - host='localhost', - grant_option=False, - escape=True): - ''' - Adds a grant to the MySQL server. - - CLI Example:: - - salt '*' mysql.grant_add 'SELECT|INSERT|UPDATE|...' 'database.*' 'frank' 'localhost' - ''' - # todo: validate grant - db = connect() - cur = db.cursor() - - query = __grant_generate(grant, database, user, host, grant_option, escape) - log.debug("Query: {0}".format(query,)) - if cur.execute( query ): - log.info("Grant '{0}' created") - return True - return False - -def grant_revoke(grant, - database, - user, - host='localhost', - grant_option=False): - ''' - Removes a grant from the MySQL server. - - CLI Example:: - - salt '*' mysql.grant_revoke 'SELECT,INSERT,UPDATE' 'database.*' 'frank' 'localhost' - ''' - # todo: validate grant - db = connect() - cur = db.cursor() - - if grant_option: - grant += ", GRANT OPTION" - query = "REVOKE %s ON %s FROM '%s'@'%s';" % (grant, database, user, host,) - log.debug("Query: {0}".format(query,)) - if cur.execute( query ): - log.info("Grant '{0}' revoked") - return True - return False From dcca8289457eb694522d9f140f887a7ddfa06583 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 20:08:30 +0000 Subject: [PATCH 480/598] trying to get better formating of return --- salt/modules/postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index 72062b4127d5..95a5df702471 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -33,7 +33,7 @@ def version(): salt '*' postgres.version ''' - return __salt__['cmd.run']('psql --version') + return __salt__['cmd.run']('psql --version').split("\n") ''' Database related actions From c9ade237c6eff5adcd38e91718975f7db347a0a4 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 20:10:46 +0000 Subject: [PATCH 481/598] cleaning up version return --- salt/modules/postgres.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index 95a5df702471..d77aa7b3102c 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -33,7 +33,8 @@ def version(): salt '*' postgres.version ''' - return __salt__['cmd.run']('psql --version').split("\n") + version_line = __salt__['cmd.run']('psql --version').split("\n")[0] + return version_line[1:] ''' Database related actions From 648a307854fb060e93e54adfcce1e324dc6c6848 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 20:11:53 +0000 Subject: [PATCH 482/598] cleaning up version return --- salt/modules/postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index d77aa7b3102c..cd5acfea4daf 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -34,7 +34,7 @@ def version(): salt '*' postgres.version ''' version_line = __salt__['cmd.run']('psql --version').split("\n")[0] - return version_line[1:] + return version_line.split(" ")[1:] ''' Database related actions From ee0df58d0e1fe9ff262196e96395976ddeb08c15 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 20:16:39 +0000 Subject: [PATCH 483/598] cleaning up version return --- salt/modules/postgres.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index cd5acfea4daf..05c13ad16281 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -34,7 +34,10 @@ def version(): salt '*' postgres.version ''' version_line = __salt__['cmd.run']('psql --version').split("\n")[0] - return version_line.split(" ")[1:] + name = version_line.split(" ")[1] + ver = version_line.split(" ")[2] + print "{0} {1}".format(name, ver) + return "{0} {1}".format(name, ver) ''' Database related actions From 957a8415093fc6bfe9dc7ad03af9f5b309a854bd Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 22:24:25 +0000 Subject: [PATCH 484/598] making progress towards a useable postgres module --- salt/modules/postgres.py | 50 ++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index 05c13ad16281..098f7dfdc23f 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -1,21 +1,16 @@ ''' -Module to provide MySQL compatibility to salt. +Module to provide Postgres compatibility to salt. -In order to connect to MySQL, certain configuration is required +In order to connect to Postgres, certain configuration is required in /etc/salt/minion on the relevant minions. Some sample configs might look like:: postgres.host: 'localhost' - postgres.port: 3306 - postgres.user: 'root' + postgres.port: '5432' + postgres.user: 'postgres' postgres.pass: '' - postgres.db: 'mysql' + postgres.db: 'postgres' -You can also use a defaults file:: - - mysql.default_file: '/etc/mysql/debian.cnf' - -Required python modules: MySQLdb ''' import logging @@ -26,8 +21,8 @@ def version(): ''' - Return the version of a MySQL server using the output - from the ``SELECT VERSION()`` query. + Return the version of a Postgres server using the output + from the ``psql --version`` cmd. CLI Example:: @@ -36,30 +31,41 @@ def version(): version_line = __salt__['cmd.run']('psql --version').split("\n")[0] name = version_line.split(" ")[1] ver = version_line.split(" ")[2] - print "{0} {1}".format(name, ver) - return "{0} {1}".format(name, ver) + return "%s %s" % (name, ver) + ''' Database related actions ''' -def db_list(user, host): +def db_list(user=None, host=None): ''' Return a list of databases of a MySQL server using the output - from the ``SHOW DATABASES`` query. + from the ``psql -l`` query. CLI Example:: salt '*' postgres.db_list ''' + if not user: + user = __opts__['postgres.user'] + if not host: + host = __opts__['postgres.host'] + + ret = [] cmd = "psql -l -U {user} -h {host}".format( user=user, host=host) + lines = [x for x in __salt__['cmd.run'](cmd).split("\n") if len(x.split("|")) == 6] + header = [x.strip() for x in lines[0].split("|")] + for line in lines[1:]: + line = [x.strip() for x in line.split("|")] + if not line[0] == "": + ret.append(zip(header[:-1], line[:-1])) - __salt__['cmd.run'](cmd) - + return ret -def db_exists(user, host, name): +def db_exists(name, user=None, host=None): ''' Checks if a database exists on the MySQL server. @@ -68,7 +74,11 @@ def db_exists(user, host, name): salt '*' mysql.db_exists 'dbname' ''' databases = __salt__['postgres.db_list'](user, host) - return name in databases + for db in databases: + if name == dict(db).get('Name'): + return True + + return False def db_create(user, host, name, **kwargs): From 35cee145937607ae33091f5a585f757cb044719e Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Sun, 18 Mar 2012 23:05:30 +0000 Subject: [PATCH 485/598] adding ability to create, delete, check existance and listing of databases --- salt/modules/postgres.py | 92 +++++++++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 21 deletions(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index 098f7dfdc23f..cc40a5ea7265 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -39,9 +39,9 @@ def version(): ''' -def db_list(user=None, host=None): +def db_list(user=None, host=None, port=None): ''' - Return a list of databases of a MySQL server using the output + Return a list of databases of a Postgres server using the output from the ``psql -l`` query. CLI Example:: @@ -52,10 +52,14 @@ def db_list(user=None, host=None): user = __opts__['postgres.user'] if not host: host = __opts__['postgres.host'] + if not port: + port = __opts__['postgres.port'] ret = [] - cmd = "psql -l -U {user} -h {host}".format( - user=user, host=host) + cmd = "psql -l -h {host} -U {user} -p {port}".format( + host=host, user=user, port=port) + + lines = [x for x in __salt__['cmd.run'](cmd).split("\n") if len(x.split("|")) == 6] header = [x.strip() for x in lines[0].split("|")] for line in lines[1:]: @@ -65,15 +69,15 @@ def db_list(user=None, host=None): return ret -def db_exists(name, user=None, host=None): +def db_exists(name, user=None, host=None, port=None): ''' - Checks if a database exists on the MySQL server. + Checks if a database exists on the Postgres server. CLI Example:: - salt '*' mysql.db_exists 'dbname' + salt '*' postgres.db_exists 'dbname' ''' - databases = __salt__['postgres.db_list'](user, host) + databases = __salt__['postgres.db_list'](user=user, host=host, port=port) for db in databases: if name == dict(db).get('Name'): return True @@ -81,28 +85,69 @@ def db_exists(name, user=None, host=None): return False -def db_create(user, host, name, **kwargs): +def db_create(name, + user=None, + host=None, + port=None, + tablespace=None, + encoding=None, + local=None, + lc_collate=None, + lc_ctype=None, + owner=None, + template=None): ''' - Adds a databases to the MySQL server. + Adds a databases to the Postgres server. CLI Example:: - salt '*' mysql.db_create 'dbname' + salt '*' postgres.db_create 'dbname' + + salt '*' postgres.db_create 'dbname' template=template_postgis + ''' # check if db exists - if db_exists(name): + if db_exists(name, user, host, port): log.info("DB '{0}' already exists".format(name,)) return False - - cmd = 'create_db {name} '.format(name) - for param, value in kwargs.iteritems(): - cmd = '{cmd} {param} {value} '.format( - cmd=cmd, param=param, value=value) - + + cmd = 'createdb {0}'.format(name) + + if tablespace: + cmd = "{0} -D {1}".format(cmd, tablespace) + + if encoding: + cmd = "{0} -E {1}".format(cmd, encoding) + + if local: + cmd = "{0} -l {1}".format(cmd, local) + + if lc_collate: + cmd = "{0} --lc-collate {1}".format(cmd, lc_collate) + + if lc_ctype: + cmd = "{0} --lc-ctype {1}".format(cmd, lc_ctype) + + if owner: + cmd = "{0} -O {1}".format(cmd, owner) + + if template: + if db_exists(template, user, host, port): + cmd = "{cmd} -T {template}".format(cmd=cmd, template=template) + else: + log.info("template '{0}' does not exist.".format(template, )) + return False + __salt__['cmd.run'](cmd) + if db_exists(name, user, host, port): + return True + else: + log.info("Failed to create DB '{0}'".format(name,)) + return False + -def db_remove(name): +def db_remove(name, user=None, host=None, port=None): ''' Removes a databases from the MySQL server. @@ -116,9 +161,14 @@ def db_remove(name): return False # db doesnt exist, proceed - cmd = 'dropdb {name} '.format(name) + cmd = 'dropdb {0}'.format(name) __salt__['cmd.run'](cmd) - + if not db_exists(name, user, host, port): + return True + else: + log.info("Failed to delete DB '{0}'.".format(name, )) + return False + ''' User related actions From 4f871ee1cbbf7b5cf7e3eb8a7d155d87459090de Mon Sep 17 00:00:00 2001 From: Nick Lang <nick@nicklang.com> Date: Sun, 18 Mar 2012 17:23:19 -0600 Subject: [PATCH 486/598] adding create user ability --- salt/modules/postgres.py | 66 ++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index cc40a5ea7265..cdbe9c9752c4 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -59,16 +59,16 @@ def db_list(user=None, host=None, port=None): cmd = "psql -l -h {host} -U {user} -p {port}".format( host=host, user=user, port=port) - lines = [x for x in __salt__['cmd.run'](cmd).split("\n") if len(x.split("|")) == 6] header = [x.strip() for x in lines[0].split("|")] for line in lines[1:]: line = [x.strip() for x in line.split("|")] - if not line[0] == "": + if not line[0] == "": ret.append(zip(header[:-1], line[:-1])) return ret + def db_exists(name, user=None, host=None, port=None): ''' Checks if a database exists on the Postgres server. @@ -85,10 +85,10 @@ def db_exists(name, user=None, host=None, port=None): return False -def db_create(name, - user=None, - host=None, - port=None, +def db_create(name, + user=None, + host=None, + port=None, tablespace=None, encoding=None, local=None, @@ -101,18 +101,18 @@ def db_create(name, CLI Example:: - salt '*' postgres.db_create 'dbname' + salt '*' postgres.db_create 'dbname' salt '*' postgres.db_create 'dbname' template=template_postgis - + ''' # check if db exists if db_exists(name, user, host, port): log.info("DB '{0}' already exists".format(name,)) return False - + cmd = 'createdb {0}'.format(name) - + if tablespace: cmd = "{0} -D {1}".format(cmd, tablespace) @@ -137,7 +137,7 @@ def db_create(name, else: log.info("template '{0}' does not exist.".format(template, )) return False - + __salt__['cmd.run'](cmd) if db_exists(name, user, host, port): @@ -149,11 +149,11 @@ def db_create(name, def db_remove(name, user=None, host=None, port=None): ''' - Removes a databases from the MySQL server. + Removes a databases from the Postgres server. CLI Example:: - salt '*' mysql.db_remove 'dbname' + salt '*' postgres.db_remove 'dbname' ''' # check if db exists if not db_exists(name): @@ -168,24 +168,44 @@ def db_remove(name, user=None, host=None, port=None): else: log.info("Failed to delete DB '{0}'.".format(name, )) return False - ''' User related actions ''' -def user_create(user, - host='localhost', - password=None, - password_hash=None): + + +def user_create(username, + user=None, + host=None, + port=None, + createdb=False, + createuser=False, + encrypted=False, + password=None): ''' - Creates a MySQL user. + Creates a Postgres user. CLI Examples:: - salt '*' mysql.user_create 'username' 'hostname' 'password - - salt '*' mysql.user_create 'username' 'hostname' password_hash='hash' + salt '*' postgres.user_create 'username' user='user' host='hostname' port='port' password='password' ''' - pass + sub_cmd = "CREATE USER {0} WITH".format(username, ) + if password: + sub_cmd = "{0} PASSWORD '{1}'".format(sub_cmd, password) + if createdb: + sub_cmd = "{0} CREATEDB".format(sub_cmd, ) + if createuser: + sub_cmd = "{0} CREATEUSER".format(sub_cmd, ) + if encrypted: + sub_cmd = "{0} ENCRYPTED".format(sub_cmd, ) + + if sub_cmd.endswith("WITH"): + sub_cmd = sub_cmd.replace(" WITH", "") + + cmd = "psql -h {host} -U {user} -p {port} -c '{sub_cmd}'".format( + host=host, user=user, port=port, sub_cmd=sub_cmd) + + return __salt__['cmd.run'](cmd) + From 0a997d6ba0437634d8ad43d7c48f4c4a40678c9c Mon Sep 17 00:00:00 2001 From: Nick Lang <nick@nicklang.com> Date: Sun, 18 Mar 2012 17:26:15 -0600 Subject: [PATCH 487/598] adding ability to get default values for connecting to postgres --- salt/modules/postgres.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index cdbe9c9752c4..4ade43455ee7 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -189,6 +189,13 @@ def user_create(username, salt '*' postgres.user_create 'username' user='user' host='hostname' port='port' password='password' ''' + if not user: + user = __opts__['postgres.user'] + if not host: + host = __opts__['postgres.host'] + if not port: + port = __opts__['postgres.port'] + sub_cmd = "CREATE USER {0} WITH".format(username, ) if password: sub_cmd = "{0} PASSWORD '{1}'".format(sub_cmd, password) From edfa02c0010965336595b1de5434c72b38d8af6d Mon Sep 17 00:00:00 2001 From: blast_hardcheese <blast@hardchee.se> Date: Sun, 18 Mar 2012 16:43:19 -0700 Subject: [PATCH 488/598] Adding some logic to handle virtual packages If aptitude is installed, it is used to check whether a virtual package has been satisfied. Aptitude awesomeness found here: http://unix.stackexchange.com/questions/14295/aptitude-what-is-the- filter-for-virtual-packages-that-has-been-provided --- salt/modules/apt.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/modules/apt.py b/salt/modules/apt.py index f33c5fe24b18..825970220fba 100644 --- a/salt/modules/apt.py +++ b/salt/modules/apt.py @@ -265,6 +265,17 @@ def list_pkgs(regex_string=""): if len(cols) and 'ii' in cols[0]: ret[cols[1]] = cols[2] + # If ret is empty at this point, check to see if the package is virtual. + # We also need aptitude past this point. + if not ret and __salt__['cmd.has_exec']('aptitude'): + cmd = ('aptitude search "{} ?virtual ?reverse-provides(?installed)"' + .format(regex_string)) + + out = __salt__['cmd.run_stdout'](cmd) + if out: + ret[regex_string] = '1' # Setting all 'installed' virtual package + # versions to '1' + return ret From 4434fe995fc8f822c7611d91adf38124bb2a92f7 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 11:57:14 -0700 Subject: [PATCH 489/598] Add a bunch of docs and a check for the git module Now it will gracefully inform the user that git isn't installed --- salt/modules/git.py | 175 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 164 insertions(+), 11 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 3f69919a2186..8d6ccc6e7424 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -18,14 +18,37 @@ def _git_getdir(cwd, user=None): cmd_toplvl = 'git rev-parse --show-toplevel' return __salt__['cmd.run'](cmd_toplvl, cwd) +def _check_git(): + ''' + Instead of requiring a minion restart after installing + git to use this module, just check and gracefully tell + the user they need to install git first. + ''' + if not __salt__['cmd.has_exec']('git'): + raise CommandNotFoundError('git') + def revision(cwd, rev='HEAD', short=False, user=None): ''' Returns the long hash of a given identifier (hash, branch, tag, HEAD, etc) - Usage:: + cwd + The path to the Git repository + + rev: HEAD + The path to the archive tarball + + short: False + Return an abbreviated SHA1 git hash + + user : (none) + Run git as a user other than what the minion runs as + + CLI Example:: salt '*' git.revision /path/to/repo mybranch ''' + _check_git() + cmd = 'git rev-parse {0}{1}'.format('--short ' if short else '', rev) result = __salt__['cmd.run_all'](cmd, cwd, runas=user) @@ -38,7 +61,19 @@ def clone(cwd, repository, opts=None, user=None): ''' Clone a new repository - Usage:: + cwd + The path to the Git repository + + repository + The git uri of the repository + + opts : (none) + Any additional options to add to the command line + + user : (none) + Run git as a user other than what the minion runs as + + CLI Example:: salt '*' git.clone /path/to/repo git://github.com/saltstack/salt.git @@ -46,6 +81,8 @@ def clone(cwd, repository, opts=None, user=None): git://github.com/saltstack/salt.git '--bare --origin github' ''' + _check_git() + if not opts: opts = '' cmd = 'git clone {0} {1} {2}'.format(repository, cwd, opts) @@ -56,6 +93,15 @@ def describe(cwd, rev='HEAD', user=None): ''' Returns the git describe string (or the SHA hash if there are no tags) for the given revision + + cwd + The path to the Git repository + + rev: HEAD + The path to the archive tarball + + user : (none) + Run git as a user other than what the minion runs as ''' cmd = 'git describe {0}'.format(rev) return __salt__['cmd.run_stdout'](cmd, cwd=cwd, runas=user) @@ -64,13 +110,33 @@ def archive(cwd, output, rev='HEAD', fmt='', prefix='', user=None): ''' Export a tarball from the repository + cwd + The path to the Git repository + + output + The path to the archive tarball + + rev: HEAD + The path to the archive tarball + + fmt: '' + Format of the resulting archive, zip and tar are commonly used + + prefix: '' + Prepend <prefix>/ to every filename in the archive + + user : (none) + Run git as a user other than what the minion runs as + If ``prefix`` is not specified it defaults to the basename of the repo directory. - Usage:: + CLI Example:: salt '*' git.archive /path/to/repo /path/to/archive.tar.gz ''' + _check_git() + basename = '{0}/'.format(os.path.basename(_git_getdir(cwd, user=user))) cmd = 'git archive{prefix}{fmt} -o {output} {rev}'.format( @@ -85,10 +151,23 @@ def fetch(cwd, opts=None, user=None): ''' Perform a fetch on the given repository - Usage:: + cwd + The path to the Git repository + + opts : (none) + Any additional options to add to the command line + + user : (none) + Run git as a user other than what the minion runs as + + CLI Example:: salt '*' git.fetch /path/to/repo '--all' + + salt '*' git.fetch cwd=/path/to/repo opts='--all' user=johnny ''' + _check_git() + if not opts: opts = '' cmd = 'git fetch {0}'.format(opts) @@ -99,19 +178,73 @@ def pull(cwd, opts=None, user=None): ''' Perform a pull on the given repository - Usage:: + cwd + The path to the Git repository + + opts : (none) + Any additional options to add to the command line + + user : (none) + Run git as a user other than what the minion runs as + + CLI Example:: - salt '*' git.pull /path/to/repo '--rebase origin master' + salt '*' git.pull /path/to/repo opts='--rebase origin master' ''' + _check_git() + if not opts: opts = '' return __salt__['cmd.run']('git pull {0}'.format(opts), cwd=cwd, runas=user) +def rebase(cwd, rev='master', opts=None, user=None): + ''' + Rebase the current branch + + cwd + The path to the Git repository + + rev + The remote branch or revision to merge into the current branch + + opts : (none) + Any additional options to add to the command line + + user : (none) + Run git as a user other than what the minion runs as + + CLI Example:: + + salt '*' git.rebase /path/to/repo master + + That is the same as: git rebase master + ''' + _check_git() + + if not opts: + opts = '' + return __salt__['cmd.run']('git rebase {0}'.format(opts), cwd=cwd, runas=user) + def checkout(cwd, rev, force=False, opts=None, user=None): ''' Checkout a given revision - Usage:: + cwd + The path to the Git repository + + rev + The remote branch or revision to merge into the current branch + + force : (none) + The remote branch or revision to merge into the current branch + + opts : (none) + Any additional options to add to the command line + + user : (none) + Run git as a user other than what the minion runs as + + CLI Examples:: salt '*' git.checkout /path/to/repo somebranch user=jeff @@ -119,6 +252,8 @@ def checkout(cwd, rev, force=False, opts=None, user=None): salt '*' git.checkout /path/to/repo 'origin/mybranch --track' ''' + _check_git() + if not opts: opts = '' cmd = 'git checkout {0} {1} {2}'.format(' -f' if force else '', rev, opts) @@ -130,16 +265,23 @@ def merge(cwd, branch='@{upstream}', opts=None, user=None): cwd The path to the Git repository + branch : @{upstream} The remote branch or revision to merge into the current branch + opts : (none) Any additional options to add to the command line - Usage:: + user : (none) + Run git as a user other than what the minion runs as + + CLI Example:: salt '*' git.fetch /path/to/repo salt '*' git.merge /path/to/repo @{upstream} ''' + _check_git() + if not opts: opts = '' cmd = 'git merge {0}{1} {2}'.format( @@ -150,11 +292,22 @@ def merge(cwd, branch='@{upstream}', opts=None, user=None): def init(cwd, opts=None, user=None): ''' - Init a new repository + Initialize a new git repository - Usage:: + cwd + The path to the Git repository - salt '*' git.init /path/to/repo.git '--bare' + opts : (none) + Any additional options to add to the command line + + user : (none) + Run git as a user other than what the minion runs as + + CLI Example:: + + salt '*' git.init /path/to/repo.git opts='--bare' ''' + _check_git() + cmd = 'git init {0} {1}'.format(cwd, opts) return __salt__['cmd.run'](cmd, runas=user) From c5bdc55d6a6fce9d17c39593bd98a57b671cbe74 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 12:01:25 -0700 Subject: [PATCH 490/598] Updating the CommandNotFound error when ran via salt-call --- salt/cli/caller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cli/caller.py b/salt/cli/caller.py index 2fd33092ff7d..2da55bdc2a0f 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -51,7 +51,7 @@ def call(self): sys.stderr.write(msg.format(fun, str(exc))) sys.exit(1) except CommandNotFoundError as exc: - msg = 'Command not found in \'{0}\': {1}\n' + msg = 'Command required for \'{0}\' not found: {1}\n' sys.stderr.write(msg.format(fun, str(exc))) sys.exit(1) if hasattr(self.minion.functions[fun], '__outputter__'): From 751256f90b8ba59c5985b71712bc24a170883c8c Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 12:04:42 -0700 Subject: [PATCH 491/598] Updating minion CommandNotFoundError message --- salt/minion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index 2cf84797e694..013aa4e0e3c7 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -240,7 +240,7 @@ def _thread_return(self, data): args, kw = salt.state.build_args(func, data['arg'], data) ret['return'] = func(*args, **kw) except CommandNotFoundError as exc: - msg = 'Command not found in \'{0}\': {1}' + msg = 'Command required for \'{0}\' not found: {1}' log.debug(msg.format(function_name, str(exc))) ret['return'] = msg.format(function_name, str(exc)) except CommandExecutionError as exc: From 831a1c58f62b2bbd518d27ce96467adc69270b41 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 12:05:14 -0700 Subject: [PATCH 492/598] Doh! forgot to import the exception --- salt/modules/git.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/git.py b/salt/modules/git.py index 8d6ccc6e7424..4850bc1b009c 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -3,6 +3,7 @@ ''' import os +from salt.exceptions import CommandNotFoundError def _git_getdir(cwd, user=None): ''' From 5d6560ea4dcd6f070fa5840e0006f80b6ec93558 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 12:09:22 -0700 Subject: [PATCH 493/598] Add a user flag to every function in the hg module This way salt users can run hg as a user != the minion's user --- salt/modules/hg.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/salt/modules/hg.py b/salt/modules/hg.py index c785059f0762..cada730bd69b 100644 --- a/salt/modules/hg.py +++ b/salt/modules/hg.py @@ -2,7 +2,7 @@ Support for the Mercurial SCM ''' -def revision(cwd, rev='tip', short=False): +def revision(cwd, rev='tip', short=False, user=None): ''' Returns the long hash of a given identifier (hash, branch, tag, HEAD, etc) @@ -14,14 +14,14 @@ def revision(cwd, rev='tip', short=False): short = ' --debug' if not short else '', rev = ' -r {0}'.format(rev)) - result = __salt__['cmd.run_all'](cmd, cwd=cwd) + result = __salt__['cmd.run_all'](cmd, cwd=cwd, runas=user) if result['retcode'] == 0: - return result['stdout'].strip('\n') + return result['stdout'] else: return '' -def describe(cwd, rev='tip'): +def describe(cwd, rev='tip', user=None): ''' Mimick git describe and return an identifier for the given revision @@ -31,11 +31,11 @@ def describe(cwd, rev='tip'): ''' cmd = "hg log -r {0} --template"\ " '{{latesttag}}-{{latesttagdistance}}-{{node|short}}'".format(rev) - desc = __salt__['cmd.run_stdout'](cmd, cwd=cwd) + desc = __salt__['cmd.run_stdout'](cmd, cwd=cwd, runas=user) return desc or revision(cwd, rev, short=True) -def archive(cwd, output, rev='tip', fmt='', prefix=''): +def archive(cwd, output, rev='tip', fmt='', prefix='', user=None): ''' Export a tarball from the repository @@ -52,9 +52,9 @@ def archive(cwd, output, rev='tip', fmt='', prefix=''): fmt = ' --type {0}'.format(fmt) if fmt else '', prefix = ' --prefix "{0}"'.format(prefix if prefix else '')) - return __salt__['cmd.run'](cmd, cwd=cwd) + return __salt__['cmd.run'](cmd, cwd=cwd, runas=user) -def pull(cwd, opts=''): +def pull(cwd, opts='', user=None): ''' Perform a pull on the given repository @@ -62,9 +62,9 @@ def pull(cwd, opts=''): salt '*' hg.pull /path/to/repo '-u' ''' - return __salt__['cmd.run']('hg pull {0}'.format(opts), cwd=cwd) + return __salt__['cmd.run']('hg pull {0}'.format(opts), cwd=cwd, runas=user) -def update(cwd, rev, force=False): +def update(cwd, rev, force=False, user=None): ''' Checkout a given revision @@ -73,9 +73,9 @@ def update(cwd, rev, force=False): salt '*' hg.update /path/to/repo somebranch ''' cmd = 'hg update {0}{1}'.format(rev, ' -C' if force else '') - return __salt__['cmd.run'](cmd, cwd=cwd) + return __salt__['cmd.run'](cmd, cwd=cwd, runas=user) -def clone(cwd, repository, opts=''): +def clone(cwd, repository, opts='', user=None): ''' Clone a new repository @@ -84,4 +84,4 @@ def clone(cwd, repository, opts=''): salt '*' hg.clone /path/to/repo https://bitbucket.org/birkenfeld/sphinx ''' cmd = 'hg clone {0} {1} {2}'.format(repository, cwd, opts) - return __salt__['cmd.run'](cmd) + return __salt__['cmd.run'](cmd, runas=user) From b361c40cefd022ce763a078ea4e4947498212089 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 12:23:16 -0700 Subject: [PATCH 494/598] Add salt.utils.check_or_die() for raising CommandNotFoundError This happens a lot in various modules and made sense to split out instead of duplicating for the git and hg modules --- salt/modules/git.py | 9 ++------- salt/utils/__init__.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 4850bc1b009c..f13bf4eb3276 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -3,6 +3,7 @@ ''' import os +from salt import utils from salt.exceptions import CommandNotFoundError def _git_getdir(cwd, user=None): @@ -20,13 +21,7 @@ def _git_getdir(cwd, user=None): return __salt__['cmd.run'](cmd_toplvl, cwd) def _check_git(): - ''' - Instead of requiring a minion restart after installing - git to use this module, just check and gracefully tell - the user they need to install git first. - ''' - if not __salt__['cmd.has_exec']('git'): - raise CommandNotFoundError('git') + utils.check_or_die('git') def revision(cwd, rev='HEAD', short=False, user=None): ''' diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index f5010769c866..9fadb18a66f1 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -9,7 +9,7 @@ import logging from calendar import month_abbr as months -from salt.exceptions import SaltClientError +from salt.exceptions import SaltClientError, CommandNotFoundError log = logging.getLogger(__name__) @@ -302,3 +302,18 @@ def required_modules_error(name, docstring): filename = os.path.basename(name).split('.')[0] msg = '\'{0}\' requires these python modules: {1}' return msg.format(filename, ', '.join(modules)) + +def check_or_die(command): + ''' + Simple convienence function for modules to use + for gracefully blowing up if a required tool is + not available in the system path. + + Lazily import salt.modules.cmdmod to avoid any + sort of circular dependencies. + ''' + import salt.modules.cmdmod + __salt__ = {'cmd.has_exec': salt.modules.cmdmod.has_exec} + + if not __salt__['cmd.has_exec'](command): + raise CommandNotFoundError(command) From 3df247ef5b1af26c559875a497392762a984a8a2 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 12:46:02 -0700 Subject: [PATCH 495/598] Fix some copy/paste mistakes --- salt/modules/git.py | 7 ++- salt/modules/hg.py | 109 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 102 insertions(+), 14 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index f13bf4eb3276..5faf01494bb9 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -4,7 +4,6 @@ import os from salt import utils -from salt.exceptions import CommandNotFoundError def _git_getdir(cwd, user=None): ''' @@ -31,7 +30,7 @@ def revision(cwd, rev='HEAD', short=False, user=None): The path to the Git repository rev: HEAD - The path to the archive tarball + The revision short: False Return an abbreviated SHA1 git hash @@ -94,7 +93,7 @@ def describe(cwd, rev='HEAD', user=None): The path to the Git repository rev: HEAD - The path to the archive tarball + The revision to describe user : (none) Run git as a user other than what the minion runs as @@ -113,7 +112,7 @@ def archive(cwd, output, rev='HEAD', fmt='', prefix='', user=None): The path to the archive tarball rev: HEAD - The path to the archive tarball + The revision to create an archive from fmt: '' Format of the resulting archive, zip and tar are commonly used diff --git a/salt/modules/hg.py b/salt/modules/hg.py index cada730bd69b..d494bac4fe9d 100644 --- a/salt/modules/hg.py +++ b/salt/modules/hg.py @@ -2,15 +2,39 @@ Support for the Mercurial SCM ''' +from salt import utils + +__outputter__ = { + 'clone': 'txt', + 'revision': 'txt', +} + +def _check_hg(): + utils.check_or_die('hg') + def revision(cwd, rev='tip', short=False, user=None): ''' Returns the long hash of a given identifier (hash, branch, tag, HEAD, etc) - Usage:: + cwd + The path to the Mercurial repository + + rev: tip + The revision + + short: False + Return an abbreviated commit hash + + user : (none) + Run hg as a user other than what the minion runs as + + CLI Example:: salt '*' hg.revision /path/to/repo mybranch ''' - cmd = 'hg id -i{short} {ref}'.format( + _check_hg() + + cmd = 'hg id -i{short} {rev}'.format( short = ' --debug' if not short else '', rev = ' -r {0}'.format(rev)) @@ -25,27 +49,59 @@ def describe(cwd, rev='tip', user=None): ''' Mimick git describe and return an identifier for the given revision - Usage:: + cwd + The path to the Mercurial repository + + rev: HEAD + The path to the archive tarball + + user : (none) + Run hg as a user other than what the minion runs as + + CLI Example:: salt '*' hg.describe /path/to/repo ''' + _check_hg() + cmd = "hg log -r {0} --template"\ " '{{latesttag}}-{{latesttagdistance}}-{{node|short}}'".format(rev) desc = __salt__['cmd.run_stdout'](cmd, cwd=cwd, runas=user) return desc or revision(cwd, rev, short=True) -def archive(cwd, output, rev='tip', fmt='', prefix='', user=None): +def archive(cwd, output, rev='tip', fmt=None, prefix=None, user=None): ''' Export a tarball from the repository + cwd + The path to the Mercurial repository + + output + The path to the archive tarball + + rev: tip + The revision to create an archive from + + fmt: (none) + Format of the resulting archive. Mercurial supports: tar, + tbz2, tgz, zip, uzip, and files formats. + + prefix: (none) + Prepend <prefix>/ to every filename in the archive + + user : (none) + Run hg as a user other than what the minion runs as + If ``prefix`` is not specified it defaults to the basename of the repo directory. - Usage:: + CLI Example:: - salt '*' hg.archive /path/to/repo /path/to/archive.tar.gz + salt '*' hg.archive /path/to/repo output=/path/to/BACKUP.tar.gz fmt=tgz ''' + _check_hg() + cmd = 'hg archive {output}{rev}{fmt}'.format( rev = ' --rev {0}'.format(rev), output = output, @@ -58,20 +114,39 @@ def pull(cwd, opts='', user=None): ''' Perform a pull on the given repository - Usage:: + cwd + The path to the Mercurial repository + + opts : (none) + Any additional options to add to the command line + + user : (none) + Run hg as a user other than what the minion runs as + + CLI Example:: salt '*' hg.pull /path/to/repo '-u' ''' + _check_hg() + return __salt__['cmd.run']('hg pull {0}'.format(opts), cwd=cwd, runas=user) def update(cwd, rev, force=False, user=None): ''' Checkout a given revision - Usage:: + cwd + The path to the Mercurial repository + + rev + The path to the archive tarball - salt '*' hg.update /path/to/repo somebranch + CLI Example:: + + salt devserver1 hg.update /path/to/repo somebranch ''' + _check_hg() + cmd = 'hg update {0}{1}'.format(rev, ' -C' if force else '') return __salt__['cmd.run'](cmd, cwd=cwd, runas=user) @@ -79,9 +154,23 @@ def clone(cwd, repository, opts='', user=None): ''' Clone a new repository - Usage:: + cwd + The path to the Git repository + + repository + The hg uri of the repository + + opts : (none) + Any additional options to add to the command line + + user : (none) + Run hg as a user other than what the minion runs as + + CLI Example:: salt '*' hg.clone /path/to/repo https://bitbucket.org/birkenfeld/sphinx ''' + _check_hg() + cmd = 'hg clone {0} {1} {2}'.format(repository, cwd, opts) return __salt__['cmd.run'](cmd, runas=user) From 0da8a7c4886cd57b167c46a70504aaad51d47bd5 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 12:58:27 -0700 Subject: [PATCH 496/598] Use salt.utils.check_or_die for the puppet module Net -4 LOC FTW --- salt/modules/puppet.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/salt/modules/puppet.py b/salt/modules/puppet.py index d7a3ce6cf40b..6d4eb3637929 100644 --- a/salt/modules/puppet.py +++ b/salt/modules/puppet.py @@ -2,7 +2,7 @@ Execute puppet routines ''' -from salt.exceptions import CommandNotFoundError +from salt import utils __outputter__ = { 'run': 'txt', @@ -18,13 +18,13 @@ def _check_puppet(): # I thought about making this a virtual module, but then I realized that I # would require the minion to restart if puppet was installed after the # minion was started, and that would be rubbish - return __salt__['cmd.has_exec']('puppetd') + utils.check_or_die('puppetd') def _check_facter(): ''' Checks if facter is installed ''' - return __salt__['cmd.has_exec']('facter') + utils.check_or_die('facter') def _format_fact(output): try: @@ -49,8 +49,7 @@ def run(tags=None): salt '*' puppet.run basefiles::edit,apache::server ''' - if not _check_puppet(): - raise CommandNotFoundError('puppetd not available') + _check_puppet(): if not tags: cmd = 'puppetd --test' @@ -71,8 +70,7 @@ def noop(tags=None): salt '*' puppet.noop web::server,django::base ''' - if not _check_puppet(): - raise CommandNotFoundError('puppetd not available') + _check_puppet(): if not tags: cmd = 'puppetd --test --noop' @@ -89,8 +87,7 @@ def facts(): salt '*' puppet.facts ''' - if not _check_facter(): - raise CommandNotFoundError('facter not available') + _check_facter(): ret = {} output = __salt__['cmd.run']('facter') @@ -114,8 +111,7 @@ def fact(name): salt '*' puppet.fact kernel ''' - if not _check_facter(): - raise CommandNotFoundError('facter not available') + _check_facter(): ret = __salt__['cmd.run']('facter {0}'.format(name)) if not ret: From af3d39917e3418c09a94a4fe60209898ceecd26d Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 13:02:40 -0700 Subject: [PATCH 497/598] A few minor module updates for missing executables This is mostly switching from raising CommandNotFoundError to using salt.utils.check_or_die('executable_name') which does an identical job in a more standardized fashion --- salt/modules/git.py | 2 +- salt/modules/upstart.py | 5 ++--- salt/modules/virtualenv.py | 7 +++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 5faf01494bb9..c27c024927fa 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -101,7 +101,7 @@ def describe(cwd, rev='HEAD', user=None): cmd = 'git describe {0}'.format(rev) return __salt__['cmd.run_stdout'](cmd, cwd=cwd, runas=user) -def archive(cwd, output, rev='HEAD', fmt='', prefix='', user=None): +def archive(cwd, output, rev='HEAD', fmt=None, prefix=None, user=None): ''' Export a tarball from the repository diff --git a/salt/modules/upstart.py b/salt/modules/upstart.py index 40a421e00bd9..1d9fff230d78 100644 --- a/salt/modules/upstart.py +++ b/salt/modules/upstart.py @@ -8,7 +8,7 @@ import os import re -from salt.exceptions import CommandNotFoundError +from salt import utils def __virtual__(): @@ -80,8 +80,7 @@ def status(name, sig=None): def _get_service_exec(): executable = 'update-rc.d' - if not __salt__['cmd.has_exec'](executable): - raise CommandNotFoundError('Missing {0}'.format(executable)) + utils.check_or_die(executable) return executable diff --git a/salt/modules/virtualenv.py b/salt/modules/virtualenv.py index e3d2fcd5e522..ced4bd753034 100644 --- a/salt/modules/virtualenv.py +++ b/salt/modules/virtualenv.py @@ -1,7 +1,7 @@ ''' Create virtualenv environments ''' -from salt.exceptions import CommandNotFoundError +from salt import utils __opts__ = { @@ -51,9 +51,8 @@ def create(path, salt '*' pip.virtualenv /path/to/new/virtualenv ''' - if not __salt__['cmd.has_exec'](venv_bin): - raise CommandNotFoundError( - "Please install {venv_bin}".format(venv_bin=venv_bin)) + # raise CommandNotFoundError if venv_bin is missing + utils.check_or_die(venv_bin) cmd = '{venv_bin} {args} {path}'.format( venv_bin=venv_bin, From bf90a00bb3d520ff3c9c820005e2c2c89fc9031c Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 14:23:55 -0700 Subject: [PATCH 498/598] Fix a small ommission when refactoring the puppet module --- salt/modules/puppet.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/puppet.py b/salt/modules/puppet.py index 6d4eb3637929..c886da13b058 100644 --- a/salt/modules/puppet.py +++ b/salt/modules/puppet.py @@ -49,7 +49,7 @@ def run(tags=None): salt '*' puppet.run basefiles::edit,apache::server ''' - _check_puppet(): + _check_puppet() if not tags: cmd = 'puppetd --test' @@ -70,7 +70,7 @@ def noop(tags=None): salt '*' puppet.noop web::server,django::base ''' - _check_puppet(): + _check_puppet() if not tags: cmd = 'puppetd --test --noop' @@ -87,7 +87,7 @@ def facts(): salt '*' puppet.facts ''' - _check_facter(): + _check_facter() ret = {} output = __salt__['cmd.run']('facter') @@ -111,7 +111,7 @@ def fact(name): salt '*' puppet.fact kernel ''' - _check_facter(): + _check_facter() ret = __salt__['cmd.run']('facter {0}'.format(name)) if not ret: From 6839e8a7fe50d376aa0093c7c7062f83242e9301 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 14:29:51 -0700 Subject: [PATCH 499/598] Adding new windows modules to the docs --- doc/ref/modules/all/index.rst | 3 +++ doc/ref/modules/all/salt.modules.win_file.rst | 6 ++++++ doc/ref/modules/all/salt.modules.win_network.rst | 6 ++++++ doc/ref/modules/all/salt.modules.win_shadow.rst | 6 ++++++ 4 files changed, 21 insertions(+) create mode 100644 doc/ref/modules/all/salt.modules.win_file.rst create mode 100644 doc/ref/modules/all/salt.modules.win_network.rst create mode 100644 doc/ref/modules/all/salt.modules.win_shadow.rst diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index 3803f7ef9524..91227d60158e 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -70,7 +70,10 @@ Full list of builtin modules virt virtualenv win_disk + win_file + win_network win_service + win_shadow win_useradd yumpkg yumpkg5 diff --git a/doc/ref/modules/all/salt.modules.win_file.rst b/doc/ref/modules/all/salt.modules.win_file.rst new file mode 100644 index 000000000000..d37930882384 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.win_file.rst @@ -0,0 +1,6 @@ +===================== +salt.modules.win_file +===================== + +.. automodule:: salt.modules.win_file + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.win_network.rst b/doc/ref/modules/all/salt.modules.win_network.rst new file mode 100644 index 000000000000..b50f9a770505 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.win_network.rst @@ -0,0 +1,6 @@ +======================== +salt.modules.win_network +======================== + +.. automodule:: salt.modules.win_network + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.win_shadow.rst b/doc/ref/modules/all/salt.modules.win_shadow.rst new file mode 100644 index 000000000000..1ba22d3ee111 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.win_shadow.rst @@ -0,0 +1,6 @@ +======================= +salt.modules.win_shadow +======================= + +.. automodule:: salt.modules.win_shadow + :members: \ No newline at end of file From fcadf900fca46ffee34eb2b9d96e4236f1b10676 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 14:35:24 -0700 Subject: [PATCH 500/598] Fix the sphinx syntax for the example rvm state --- salt/states/rvm.py | 190 ++++++++++++++++++++++----------------------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/salt/states/rvm.py b/salt/states/rvm.py index cc6835503b6c..a9526ce57af9 100644 --- a/salt/states/rvm.py +++ b/salt/states/rvm.py @@ -12,101 +12,101 @@ .. code-block:: yaml -rvm: - group: - - present - user: - - present - - gid: rvm - - home: /home/rvm - - require: - - group: rvm - -rvm-deps: - pkg: - - installed - - names: - - bash - - coreutils - - gzip - - bzip2 - - gawk - - sed - - curl - - git-core - - subversion - - sudo - -mri-deps: - pkg: - - installed - - names: - - build-essential - - openssl - - libreadline6 - - libreadline6-dev - - curl - - git-core - - zlib1g - - zlib1g-dev - - libssl-dev - - libyaml-dev - - libsqlite3-0 - - libsqlite3-dev - - sqlite3 - - libxml2-dev - - libxslt1-dev - - autoconf - - libc6-dev - - libncurses5-dev - - automake - - libtool - - bison - - subversion - - ruby - -jruby-deps: - pkg: - - installed - - names: - - curl - - g++ - - openjdk-6-jre-headless - -ruby-1.9.2: - rvm: - - installed - - default: True - - runas: rvm - - require: - - pkg: rvm-deps - - pkg: mri-deps - - user: rvm - -jruby: - rvm: - - installed - - runas: rvm - - require: - - pkg: rvm-deps - - pkg: jruby-deps - - user: rvm - -jgemset: - rvm: - - gemset_present - - ruby: jruby - - runas: rvm - - require: - - rvm: jruby - -mygemset: - rvm: - - gemset_present - - ruby: ruby-1.9.2 - - runas: rvm - - require: - - rvm: ruby-1.9.2 + rvm: + group: + - present + user: + - present + - gid: rvm + - home: /home/rvm + - require: + - group: rvm + + rvm-deps: + pkg: + - installed + - names: + - bash + - coreutils + - gzip + - bzip2 + - gawk + - sed + - curl + - git-core + - subversion + - sudo + + mri-deps: + pkg: + - installed + - names: + - build-essential + - openssl + - libreadline6 + - libreadline6-dev + - curl + - git-core + - zlib1g + - zlib1g-dev + - libssl-dev + - libyaml-dev + - libsqlite3-0 + - libsqlite3-dev + - sqlite3 + - libxml2-dev + - libxslt1-dev + - autoconf + - libc6-dev + - libncurses5-dev + - automake + - libtool + - bison + - subversion + - ruby + + jruby-deps: + pkg: + - installed + - names: + - curl + - g++ + - openjdk-6-jre-headless + + ruby-1.9.2: + rvm: + - installed + - default: True + - runas: rvm + - require: + - pkg: rvm-deps + - pkg: mri-deps + - user: rvm + + jruby: + rvm: + - installed + - runas: rvm + - require: + - pkg: rvm-deps + - pkg: jruby-deps + - user: rvm + + jgemset: + rvm: + - gemset_present + - ruby: jruby + - runas: rvm + - require: + - rvm: jruby + + mygemset: + rvm: + - gemset_present + - ruby: ruby-1.9.2 + - runas: rvm + - require: + - rvm: ruby-1.9.2 """ import re From 0d0f20667e7ea0a8db6bd03f1eb18eb1171f6ffb Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 14:54:47 -0700 Subject: [PATCH 501/598] More minor updates --- salt/modules/git.py | 2 +- salt/modules/win_disk.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index c27c024927fa..7474d38c5cc1 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -114,7 +114,7 @@ def archive(cwd, output, rev='HEAD', fmt=None, prefix=None, user=None): rev: HEAD The revision to create an archive from - fmt: '' + fmt: (none) Format of the resulting archive, zip and tar are commonly used prefix: '' diff --git a/salt/modules/win_disk.py b/salt/modules/win_disk.py index 31af7e163d23..fa8d4f284bd7 100644 --- a/salt/modules/win_disk.py +++ b/salt/modules/win_disk.py @@ -1,11 +1,11 @@ ''' Module for gathering disk information on Windows ''' -is_windows = True try: import ctypes import string import win32api + is_windows = True except ImportError: is_windows = False @@ -37,24 +37,24 @@ def usage(): try: sectorspercluster, bytespersector, freeclusters, totalclusters =\ win32api.GetDiskFreeSpace('{0}:\\'.format(drive)) - totalsize = sectorspercluster * bytespersector * totalclusters - available_space = sectorspercluster * bytespersector * freeclusters + totalsize = sectorspercluster * bytespersector * totalclusters + available_space = sectorspercluster * bytespersector * freeclusters used = totalsize - available_space capacity = int(used / float(totalsize) * 100) ret['{0}:\\'.format(drive)] = { 'filesystem': '{0}:\\'.format(drive), - '1K-blocks': totalsize, - 'used': used, + '1K-blocks': totalsize, + 'used': used, 'available': available_space, 'capacity': '{0}%'.format(capacity), } except: ret['{0}:\\'.format(drive)] = { 'filesystem': '{0}:\\'.format(drive), - '1K-blocks': None, - 'used': None, + '1K-blocks': None, + 'used': None, 'available': None, - 'capacity': None, + 'capacity': None, } return ret From fe5ad3289be7ab6d13e3ca551ef13d7b6fdc0f75 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 14:57:41 -0700 Subject: [PATCH 502/598] Some doc style nits from @blast_hardcheese Thanks! --- salt/modules/git.py | 50 ++++++++++++++++++++++----------------------- salt/modules/hg.py | 42 ++++++++++++++++++++++--------------- 2 files changed, 51 insertions(+), 41 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 7474d38c5cc1..0ce026a98ad8 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -35,7 +35,7 @@ def revision(cwd, rev='HEAD', short=False, user=None): short: False Return an abbreviated SHA1 git hash - user : (none) + user : None Run git as a user other than what the minion runs as CLI Example:: @@ -62,10 +62,10 @@ def clone(cwd, repository, opts=None, user=None): repository The git uri of the repository - opts : (none) + opts : None Any additional options to add to the command line - user : (none) + user : None Run git as a user other than what the minion runs as CLI Example:: @@ -95,7 +95,7 @@ def describe(cwd, rev='HEAD', user=None): rev: HEAD The revision to describe - user : (none) + user : None Run git as a user other than what the minion runs as ''' cmd = 'git describe {0}'.format(rev) @@ -114,13 +114,13 @@ def archive(cwd, output, rev='HEAD', fmt=None, prefix=None, user=None): rev: HEAD The revision to create an archive from - fmt: (none) + fmt: None Format of the resulting archive, zip and tar are commonly used - prefix: '' + prefix : None Prepend <prefix>/ to every filename in the archive - user : (none) + user : None Run git as a user other than what the minion runs as If ``prefix`` is not specified it defaults to the basename of the repo @@ -149,10 +149,10 @@ def fetch(cwd, opts=None, user=None): cwd The path to the Git repository - opts : (none) + opts : None Any additional options to add to the command line - user : (none) + user : None Run git as a user other than what the minion runs as CLI Example:: @@ -176,10 +176,10 @@ def pull(cwd, opts=None, user=None): cwd The path to the Git repository - opts : (none) + opts : None Any additional options to add to the command line - user : (none) + user : None Run git as a user other than what the minion runs as CLI Example:: @@ -199,13 +199,13 @@ def rebase(cwd, rev='master', opts=None, user=None): cwd The path to the Git repository - rev - The remote branch or revision to merge into the current branch + rev : master + The revision to rebase onto the current branch - opts : (none) + opts : None Any additional options to add to the command line - user : (none) + user : None Run git as a user other than what the minion runs as CLI Example:: @@ -228,15 +228,15 @@ def checkout(cwd, rev, force=False, opts=None, user=None): The path to the Git repository rev - The remote branch or revision to merge into the current branch + The remote branch or revision to checkout - force : (none) - The remote branch or revision to merge into the current branch + force : False + Force a checkout even if there might be overwritten changes - opts : (none) + opts : None Any additional options to add to the command line - user : (none) + user : None Run git as a user other than what the minion runs as CLI Examples:: @@ -245,7 +245,7 @@ def checkout(cwd, rev, force=False, opts=None, user=None): salt '*' git.checkout /path/to/repo opts='testbranch -- conf/file1 file2' - salt '*' git.checkout /path/to/repo 'origin/mybranch --track' + salt '*' git.checkout /path/to/repo rev=origin/mybranch opts=--track ''' _check_git() @@ -264,10 +264,10 @@ def merge(cwd, branch='@{upstream}', opts=None, user=None): branch : @{upstream} The remote branch or revision to merge into the current branch - opts : (none) + opts : None Any additional options to add to the command line - user : (none) + user : None Run git as a user other than what the minion runs as CLI Example:: @@ -292,10 +292,10 @@ def init(cwd, opts=None, user=None): cwd The path to the Git repository - opts : (none) + opts : None Any additional options to add to the command line - user : (none) + user : None Run git as a user other than what the minion runs as CLI Example:: diff --git a/salt/modules/hg.py b/salt/modules/hg.py index d494bac4fe9d..c163c3118f0c 100644 --- a/salt/modules/hg.py +++ b/salt/modules/hg.py @@ -25,7 +25,7 @@ def revision(cwd, rev='tip', short=False, user=None): short: False Return an abbreviated commit hash - user : (none) + user : None Run hg as a user other than what the minion runs as CLI Example:: @@ -52,10 +52,10 @@ def describe(cwd, rev='tip', user=None): cwd The path to the Mercurial repository - rev: HEAD + rev: tip The path to the archive tarball - user : (none) + user : None Run hg as a user other than what the minion runs as CLI Example:: @@ -83,14 +83,14 @@ def archive(cwd, output, rev='tip', fmt=None, prefix=None, user=None): rev: tip The revision to create an archive from - fmt: (none) + fmt: None Format of the resulting archive. Mercurial supports: tar, tbz2, tgz, zip, uzip, and files formats. - prefix: (none) + prefix : None Prepend <prefix>/ to every filename in the archive - user : (none) + user : None Run hg as a user other than what the minion runs as If ``prefix`` is not specified it defaults to the basename of the repo @@ -98,7 +98,7 @@ def archive(cwd, output, rev='tip', fmt=None, prefix=None, user=None): CLI Example:: - salt '*' hg.archive /path/to/repo output=/path/to/BACKUP.tar.gz fmt=tgz + salt '*' hg.archive /path/to/repo output=/tmp/archive.tgz fmt=tgz ''' _check_hg() @@ -110,17 +110,17 @@ def archive(cwd, output, rev='tip', fmt=None, prefix=None, user=None): return __salt__['cmd.run'](cmd, cwd=cwd, runas=user) -def pull(cwd, opts='', user=None): +def pull(cwd, opts=None, user=None): ''' Perform a pull on the given repository cwd The path to the Mercurial repository - opts : (none) + opts : None Any additional options to add to the command line - user : (none) + user : None Run hg as a user other than what the minion runs as CLI Example:: @@ -129,17 +129,25 @@ def pull(cwd, opts='', user=None): ''' _check_hg() + if not opts: + opts = '' return __salt__['cmd.run']('hg pull {0}'.format(opts), cwd=cwd, runas=user) def update(cwd, rev, force=False, user=None): ''' - Checkout a given revision + Update to a given revision cwd The path to the Mercurial repository rev - The path to the archive tarball + The revision to update to + + force : False + Force an update + + user : None + Run hg as a user other than what the minion runs as CLI Example:: @@ -150,20 +158,20 @@ def update(cwd, rev, force=False, user=None): cmd = 'hg update {0}{1}'.format(rev, ' -C' if force else '') return __salt__['cmd.run'](cmd, cwd=cwd, runas=user) -def clone(cwd, repository, opts='', user=None): +def clone(cwd, repository, opts=None, user=None): ''' Clone a new repository cwd - The path to the Git repository + The path to the Mercurial repository repository The hg uri of the repository - opts : (none) + opts : None Any additional options to add to the command line - user : (none) + user : None Run hg as a user other than what the minion runs as CLI Example:: @@ -172,5 +180,7 @@ def clone(cwd, repository, opts='', user=None): ''' _check_hg() + if not opts: + opts = '' cmd = 'hg clone {0} {1} {2}'.format(repository, cwd, opts) return __salt__['cmd.run'](cmd, runas=user) From 69587db577f34268b2e072e867088d61679f02d1 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 15:04:08 -0700 Subject: [PATCH 503/598] Add info about the new windows modules --- doc/topics/releases/0.9.8.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 07cc73008253..c4196d204b58 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -245,12 +245,17 @@ Module Updates This is a list of notable, but non-exhaustive updates with new and existing modules. +Windows support has seen a flurry of support this release cycle. We've gained +all new :doc:`file </ref/modules/all/salt.modules.win_file>`, +:doc:`network </ref/modules/all/salt.modules.win_network>`, and +:doc:`shadow </ref/modules/all/salt.modules.win_shadow>` modules. Please note +that these are still a work in progress. + For our ruby users, new :doc:`rvm </ref/modules/all/salt.modules.rvm>` and :doc:`gem </ref/modules/all/salt.modules.gem>` modules have been added along with the :doc:`associated </ref/states/all/salt.states.rvm>` :doc:`states </ref/states/all/salt.states.gem>` - The :doc:`virt </ref/modules/all/salt.modules.virt>` module gained basic Xen support. The :doc:`yum </ref/modules/all/salt.modules.yumpkg5>` From 9ebeb74a5453663562712d8f493e3c67b973599d Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 16:05:36 -0700 Subject: [PATCH 504/598] salt-call shouldn't run without arguments Reported-by: Devon Stewart <blast@hardchee.se> Also, make sure to print proper usage. --- salt/cli/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 00ab7a731f65..9faead748e66 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -208,7 +208,7 @@ def __parse(self): # Catch invalid invocations of salt such as: salt run if len(args) <= 1: parser.print_help() - parser.exit() + parser.exit(1) if opts['list']: opts['tgt'] = args[0].split(',') @@ -457,9 +457,10 @@ def __parse(self): if v is not None: opts[k] = v + # salt-cp needs arguments if len(args) <= 1: parser.print_help() - parser.exit() + parser.exit(1) if opts['list']: opts['tgt'] = args[0].split(',') @@ -635,7 +636,8 @@ def __parse(self): ''' Parse the command line arguments ''' - parser = optparse.OptionParser(version="%%prog %s" % VERSION) + usage = "%prog [options] <function> [arguments]" + parser = optparse.OptionParser(version='%%prog %s'.format(VERSION), usage=usage) parser.add_option('-g', '--grains', @@ -710,8 +712,9 @@ def __parse(self): opts['fun'] = args[0] opts['arg'] = args[1:] else: - opts['fun'] = '' - opts['arg'] = [] + # salt-call should not ever be called without arguments + parser.print_help() + parser.exit(1) verify_env([opts['pki_dir'], opts['cachedir'], From b312ced8a90d1caaaf45c9609cca81294b50d9d9 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 16:18:47 -0700 Subject: [PATCH 505/598] Update the troubleshooting documentation --- doc/topics/troubleshooting/index.rst | 8 ++++++ .../troubleshooting/yaml_idiosyncrasies.rst | 26 +++++++++---------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/doc/topics/troubleshooting/index.rst b/doc/topics/troubleshooting/index.rst index 73789364c33e..6cc14530b479 100644 --- a/doc/topics/troubleshooting/index.rst +++ b/doc/topics/troubleshooting/index.rst @@ -118,3 +118,11 @@ Or with the following salt state: sysctl: - present - value: 4096 87380 16777216 + + +Common YAML Gotchas +=================== + +An extensive list of +:doc:`yaml idiosyncrasies</topics/troubleshooting/yaml_idiosyncrasies>` +has been compiled. diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index 4bf71164bafc..f430ded6fc20 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -2,35 +2,35 @@ YAML Idiosyncrasies =================== -One of Salt's strength, the use of existing serialization systems for +One of Salt's strengths, the use of existing serialization systems for representing sls data, can also backfire. YAML is a general purpose system and there are a number of things that would seem to make sense in an sls -file that cause YAML issues. It is wise to be aware of these issues, while +file that cause YAML issues. It is wise to be aware of these issues. While reports or running into them are generally rare they can still crop up at unexpected times. Spaces vs Tabs ============== -Yaml uses spaces, period, do not use tabs in your sls files! If strange +Yaml uses spaces, period. Do not use tabs in your sls files! If strange errors are coming up in rendering sls files, make sure to check that -no tabs have crept in! +no tabs have crept in! In vi / vim, you can check with ``:se spell``. Indentation =========== -The suggested syntax for Yaml files is to use 2 spaces for indentation, -but Yaml will follow whatever indentation system that the individual file -uses. Generally 2 space indentation works very well for sls files given -the fact that the represented data is uniform and not deeply nested. +The suggested syntax for YAML files is to use 2 spaces for indentation, +but YAML will follow whatever indentation system that the individual file +uses. Indentation of two spaces works very well for sls files given the +fact that the data is uniform and not deeply nested. -Nested Dicts (key-value) +Nested Dicts (key=value) ------------------------ When dicts are more deeply nested they no longer follow the same indentation logic. This is rarely something that comes up in Salt, since deeply nested options like these are discouraged when making state modules, but some do -exist. A good example is the context and default options in the file.managed -state: +exist. A good example is the context and default options in the +:doc:`file.managed</ref/states/all/salt.states.file>` state: .. code-block:: yaml @@ -50,7 +50,7 @@ state: Notice that the spacing used is 2 spaces, and that when defining the context and defaults options there is a 4 space indent. If only a 2 space indent is -used then the information will not be correctly loaded. If using double spacing +used then the information will not be loaded correctly. If using double spacing is not desirable, then a deeply nested dict can be declared with curly braces: .. code-block:: yaml @@ -89,7 +89,7 @@ This is best explained when setting the mode for a file: - mode: 644 Salt manages this well, since the mode is passed as 644, but if the mode is -zero padded as 0644, then it is read by Yaml as an integer and evaluated as +zero padded as 0644, then it is read by YAML as an integer and evaluated as a hexadecimal value, 0644 becomes 420. Therefore, if the file mode is preceded by a 0 then it needs to be passed as a string: From 8ac5c2f6c63905791280ae931fa98ffdc6bdd799 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 16:29:24 -0700 Subject: [PATCH 506/598] Adding blurb on example salt states to the community docs --- doc/topics/community.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/topics/community.rst b/doc/topics/community.rst index f9a4faf5b5e1..9cff2f656c3b 100644 --- a/doc/topics/community.rst +++ b/doc/topics/community.rst @@ -54,6 +54,15 @@ http://red45.wordpress.com/ .. _`The Red45`: http://red45.wordpress.com/ + +Example Salt States +=================== +The official ``salt-states`` repository is: +https://github.com/saltstack/salt-states + +Another good example from one of our users is: +https://github.com/blast-hardcheese/blast-salt-states + Follow on ohloh =============== From 2908f18b04f3397dd11641644572bff5a555dad1 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 16:44:06 -0700 Subject: [PATCH 507/598] Added doc bits about firewall config on Ubuntu and Debian --- doc/topics/tutorials/firewall.rst | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/doc/topics/tutorials/firewall.rst b/doc/topics/tutorials/firewall.rst index e19126c937c5..3e1787b01028 100644 --- a/doc/topics/tutorials/firewall.rst +++ b/doc/topics/tutorials/firewall.rst @@ -1,6 +1,6 @@ -==================== -Opening the Firewall -==================== +================================ +Opening the Firewall up for Salt +================================ The Salt master communicates with the minions using an AES-encrypted ZeroMQ connection. These communications are done over ports 4505 and 4506, which need @@ -25,11 +25,11 @@ some of the more common locations, but your mileage may vary. **Arch Linux** :: - /etc/iptables/iptables.ruls + /etc/iptables/iptables.rules -**Debian/Ubuntu** :: +**Debian** - ??? +Follow these instructions: http://wiki.debian.org/iptables Once you've found your firewall rules, you'll need to add the two lines below to allow traffic on ``tcp/4505`` and ``tcp/4506``: @@ -39,6 +39,15 @@ to allow traffic on ``tcp/4505`` and ``tcp/4506``: + -A INPUT -m state --state new -m tcp -p tcp --dport 4505 -j ACCEPT + -A INPUT -m state --state new -m tcp -p tcp --dport 4506 -j ACCEPT +**Ubuntu** + +Create a file named ``/etc/ufw/applications.d/salt-master`` :: + + [Salt Master] + title=Salt master + description=Salt is a remote execution and configuration management tool. + ports=4205,4206/tcp + pf.conf ======= From d8987c6cc65754492766ddcdc67f8c4e534aca52 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 16:49:01 -0700 Subject: [PATCH 508/598] rST-ify some text and link to the firewall doc in troubleshooting --- doc/topics/troubleshooting/index.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/topics/troubleshooting/index.rst b/doc/topics/troubleshooting/index.rst index 6cc14530b479..5f43ebe13236 100644 --- a/doc/topics/troubleshooting/index.rst +++ b/doc/topics/troubleshooting/index.rst @@ -34,6 +34,9 @@ You can check port connectivity from the minion with the nc command: # nc -v -z salt.master.ip 4505 # nc -v -z salt.master.ip 4506 +There is also a :doc:`firewall configuration</topics/tutorials/firewall>` +document that might help as well. + Using salt-call =============== @@ -81,9 +84,9 @@ Salt Master Stops Responding There are known bugs with ZeroMQ less than 2.1.11 which can cause the salt master to not respond properly. If you're running ZeroMQ greater than or equal -to 2.1.9, you can work around the bug by setting the sysctl net.core.rmem_max -and net.core.wmem_max to 16777216. Next set the third field in net.ipv4.tcp_rmem -and net.ipv4.tcp_wmem to at least 16777216. +to 2.1.9, you can work around the bug by setting the sysctls +``net.core.rmem_max`` and ``net.core.wmem_max`` to 16777216. Next set the third +field in ``net.ipv4.tcp_rmem`` and ``net.ipv4.tcp_wmem`` to at least 16777216. You can do it manually with something like: From 260f976d73d2540936097ab7e4d6f119c6d5362c Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 16:49:31 -0700 Subject: [PATCH 509/598] More doc updates --- doc/contents.rst | 3 +++ doc/ref/modules/all/salt.modules.debconf.rst | 6 ------ doc/topics/configuration.rst | 4 ++++ 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 doc/ref/modules/all/salt.modules.debconf.rst diff --git a/doc/contents.rst b/doc/contents.rst index 8ef6c75b7b40..b58d98a19140 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -12,6 +12,9 @@ Full Table of Contents topics/targeting/index topics/tutorials/modules topics/tutorials/states* + topics/pillar/index + topics/jobs/index + topics/troubleshooting/index topics/community ref/index diff --git a/doc/ref/modules/all/salt.modules.debconf.rst b/doc/ref/modules/all/salt.modules.debconf.rst deleted file mode 100644 index b05ffe9a68bb..000000000000 --- a/doc/ref/modules/all/salt.modules.debconf.rst +++ /dev/null @@ -1,6 +0,0 @@ -==================== -salt.modules.debconf -==================== - -.. automodule:: salt.modules.debconf - :members: \ No newline at end of file diff --git a/doc/topics/configuration.rst b/doc/topics/configuration.rst index 732be71b0fd5..ed288e0a6f2c 100644 --- a/doc/topics/configuration.rst +++ b/doc/topics/configuration.rst @@ -54,6 +54,10 @@ Running Salt line or assign ``user`` in the :doc:`configuration file</ref/configuration/master>`. + +There is also a full :doc:`troubleshooting guide</topics/troubleshooting/index>` +available. + Manage Salt public keys ======================= From 5280efafcaa54c905609ed8f51cd4517cc5fa0ec Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 17:28:04 -0700 Subject: [PATCH 510/598] Big update to the "Starting States" doc --- doc/topics/tutorials/starting_states.rst | 49 ++++++++++++------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/doc/topics/tutorials/starting_states.rst b/doc/topics/tutorials/starting_states.rst index b20572bf3f65..a7d672478195 100644 --- a/doc/topics/tutorials/starting_states.rst +++ b/doc/topics/tutorials/starting_states.rst @@ -5,26 +5,25 @@ How Do I Use Salt States? Simplicity, Simplicity, Simplicity Many of the most powerful and useful engineering solutions are founded on -simple principals, the Salt SLS system strives to do just that. +simple principals, the Salt SLS system strives to do just that. K.I.S.S. The core of the Salt State system is the SLS, or the SaLt State file. The SLS is a representation of the state in which a system should be in, and is set up -to contain this data in the most simple way possible. +to contain this data simply. This is often called configuration management. It is All Just Data =================== Before delving into the particulars, it will help to understand that the SLS is just a data structure under the hood. While understanding that the SLS is -just a data structure is not at all critical to understand to make use Salt States, -it should help bolster the understanding of where the real power is. +just a data structure is not at all critical to understand to make use Salt +States, it should help bolster the understanding of where the real power is. SLS files are therefore, in reality, just dictionaries, lists, strings and -numbers. By using this approach Salt can be much more flexible, and as someone -writes more SLS files it becomes clear exactly what is being written. The result -is a system that is easy to understand, yet grows with the needs of the admin -or developer, offering simple constructs that grow to encompass the most -complicated needs. +numbers. By using this approach Salt can be much more flexible. As someone +writes more state files, it becomes clear exactly what is being written. The +result is a system that is easy to understand, yet grows with the needs of +the admin or developer. In the section titled "State Data Structures" a reference exists, explaining in depth how the data is laid out. @@ -35,7 +34,7 @@ Default Data - YAML By default Salt represents the SLS data in what is one of the simplest serialization formats available - YAML. -A typical, small SLS file will often look like this in YAML: +A typical SLS file will often look like this in YAML: .. code-block:: yaml :linenos: @@ -63,14 +62,14 @@ lines are the function to run. This function defines what state the named package and service should be in. Here the package is to be installed, and the service should be running. -Finally, on line 6, is the word ``require``, this is called a Requisite -Statement, and it makes sure that the apache service is only started after +Finally, on line 6, is the word ``require``. This is called a Requisite +Statement, and it makes sure that the Apache service is only started after the successful installation of the apache package. Adding Configs and Users ======================== -When setting up a service like an apache server many more components may +When setting up a service like an apache web server, many more components may need to be added. The apache configuration file will most likely be managed, and a user and group may need to be set up. @@ -130,14 +129,14 @@ config file will also trigger a restart of the respective service. Moving Beyond a Single SLS ========================== -When setting up Salt States more than one SLS will need to be used, the above +When setting up Salt States, more than one SLS will need to be used. The above examples were just in a single SLS file, but more than one SLS file can be combined to build out a State Tree. The above example also references a file -with a strange source - salt://apache/httpd.conf, that file will need to be -available as well. +with a strange source - ``salt://apache/httpd.conf``. That file will need to +be available as well. The SLS files are laid out in a directory on the salt master. Files are laid -out as just files, an sls is just a file and files to download are just files. +out as just files. A sls is just a file and files to download are just files. The apache example would be laid out in the root of the salt file server like this: :: @@ -268,7 +267,7 @@ These examples will add more watchers to apache and change the ssh banner. pkg: - installed -The custom-server.sls file uses the extend statement to overwrite where the +The ``custom-server.sls`` file uses the extend statement to overwrite where the banner is being downloaded from, and therefore changing what file is being used to configure the banner. @@ -298,18 +297,18 @@ python or ``py`` renderer. The ``py`` renderer allows for SLS files to be written in pure python, allowing for the utmost level of flexibility and power when preparing SLS data. -Geting to Know the Default - yaml_jinja +Getting to Know the Default - yaml_jinja --------------------------------------- The default renderer - ``yaml_jinja``, allows for the use of the jinja templating system. A guide to the jinja templating system can be found here: -<link to the jinga templating docs page>. +http://jinja.pocoo.org/docs When working with renderers a few very useful bits of data are passed in. In the case of templating engine based renderers two critical components are -available, ``salt`` and ``grains``. The salt object allows for any salt -function to be called from within the template, and grains allows for the -grains to be accessed from within the template. A few examples are in order: +available, ``salt``, ``grains``, and ``pillar``. The salt object allows for +any salt function to be called from within the template, and grains allows for +the grains to be accessed from within the template. A few examples: ``/apache/init.sls:`` @@ -353,7 +352,7 @@ grains to be accessed from within the template. A few examples are in order: - group: root - mode: 644 -This example is simple, if the os grain states that the operating system is +This example is simple. If the ``os`` grain states that the operating system is Red Hat, then the name of the apache package and service needs to be httpd. A more aggressive way to use Jinja can be found here, in a module to set up @@ -453,7 +452,7 @@ This is a very simple example, the first line has a SLS shebang line that tells Salt to not use the default renderer, but to use the ``py`` renderer. Then the run function is defined, the return value from the run function must be a Salt friendly data structure, or better known as a Salt -``HighState`` data structure. +:doc:`HighState data structure</ref/states/highstate>`. This python example would look like this if it were written in YAML: From 4119f4f9dc8f5d8173c2c8133f75491c2c169ea3 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 17:38:16 -0700 Subject: [PATCH 511/598] Fix some sphinx warnings and link docs better --- doc/contents.rst | 1 + doc/index.rst | 7 ++++--- doc/topics/troubleshooting/index.rst | 5 +++-- doc/topics/tutorials/starting_states.rst | 3 ++- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/contents.rst b/doc/contents.rst index b58d98a19140..b1be6b281af8 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -11,6 +11,7 @@ Full Table of Contents topics/configuration topics/targeting/index topics/tutorials/modules + topics/tutorials/starting_states topics/tutorials/states* topics/pillar/index topics/jobs/index diff --git a/doc/index.rst b/doc/index.rst index e26ea712c05d..ec5ed2534648 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -46,9 +46,10 @@ Now that you have the basics out of the way, learn to use Salt to configure your servers. This is widely known as :term:`configuration management` — installing packages, configuring users and services, and much more. -1. :doc:`Basic config management <topics/tutorials/states_pt1>` -2. :doc:`Less basic config management <topics/tutorials/states_pt2>` -3. :doc:`Advanced techniques <topics/tutorials/states_pt3>` +1. :doc:`Getting Started with States<topics/tutorials/starting_states>` +2. :doc:`Basic config management <topics/tutorials/states_pt1>` +3. :doc:`Less basic config management <topics/tutorials/states_pt2>` +4. :doc:`Advanced techniques <topics/tutorials/states_pt3>` Salt in depth ============= diff --git a/doc/topics/troubleshooting/index.rst b/doc/topics/troubleshooting/index.rst index 5f43ebe13236..51977e47a6c3 100644 --- a/doc/topics/troubleshooting/index.rst +++ b/doc/topics/troubleshooting/index.rst @@ -41,11 +41,12 @@ document that might help as well. Using salt-call =============== -The salt-call command was originally developed for aiding in the development +The ``salt-call`` command was originally developed for aiding in the development of new salt modules. Since then many applications have arisen for the salt-call command that is bundled with the salt minion. These range from the original intent of the salt-call, development assistance, to gathering large amounts of -data from complex calls like state.highstate. +data from complex calls like +:doc:`state.highstate</ref/modules/all/salt.modules.state>`. When developing the state tree it is generally recommended to invoke state.highstate with salt-call, this displays a great deal more information diff --git a/doc/topics/tutorials/starting_states.rst b/doc/topics/tutorials/starting_states.rst index a7d672478195..64e7e280ca70 100644 --- a/doc/topics/tutorials/starting_states.rst +++ b/doc/topics/tutorials/starting_states.rst @@ -298,7 +298,7 @@ written in pure python, allowing for the utmost level of flexibility and power when preparing SLS data. Getting to Know the Default - yaml_jinja ---------------------------------------- +---------------------------------------- The default renderer - ``yaml_jinja``, allows for the use of the jinja templating system. A guide to the jinja templating system can be found here: @@ -470,3 +470,4 @@ This clearly illustrates, that not only is using the YAML renderer a wise decision as the default, but that unbridled power can be obtained where needed by using a pure python SLS. +Now onto the :doc:`States tutorial, part 1</topics/tutorials/states_pt1>`. From 9ce7db97108e8b6af3f1eb912f9c1fcadfb828ee Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 17:55:53 -0700 Subject: [PATCH 512/598] Fix a busted link in the docs to grain info --- doc/topics/tutorials/states_pt3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/tutorials/states_pt3.rst b/doc/topics/tutorials/states_pt3.rst index d12cb9f044aa..d87cfa8134aa 100644 --- a/doc/topics/tutorials/states_pt3.rst +++ b/doc/topics/tutorials/states_pt3.rst @@ -49,7 +49,7 @@ Using Grains in SLS modules =========================== Often times a state will need to behave differently on different systems. -:doc:`Salt grains </ref/grains>` can be used from within sls modules. An object +:doc:`Salt grains </topics/targeting/grains>` can be used from within sls modules. An object called ``grains`` is made available in the template context: .. code-block:: yaml From 65fecfd79f0d6e81423039b7c75501651e668194 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 17:56:06 -0700 Subject: [PATCH 513/598] Fix a few more sphinx warnings --- doc/ref/renderers/all/index.rst | 10 +++++----- salt/modules/test.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/ref/renderers/all/index.rst b/doc/ref/renderers/all/index.rst index 7ac25a803467..61023b76a1ee 100644 --- a/doc/ref/renderers/all/index.rst +++ b/doc/ref/renderers/all/index.rst @@ -10,8 +10,8 @@ Full list of builtin renderers :toctree: :template: autosummary.rst.tmpl - json_jinja - json_mako - yaml_jinja - yaml_mako - py + salt.renderers.json_jinja + salt.renderers.json_mako + salt.renderers.yaml_jinja + salt.renderers.yaml_mako + salt.renderers.py diff --git a/salt/modules/test.py b/salt/modules/test.py index c6d16745cef1..09adea3be609 100644 --- a/salt/modules/test.py +++ b/salt/modules/test.py @@ -89,7 +89,7 @@ def cross_test(func, args=None): def kwarg(**kwargs): ''' - Print out the data passed into the function **kwargs, this is used to + Print out the data passed into the function ``**kwargs``, this is used to both test the publication data and cli kwarg passing, but also to display the information available within the publication data. From 1a6e48de89358ad61dbbf899fe0d70508530168a Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 18:04:25 -0700 Subject: [PATCH 514/598] Fix some sphinx warnings in the jobs docs --- doc/topics/jobs/index.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/doc/topics/jobs/index.rst b/doc/topics/jobs/index.rst index 7e3eb084eedc..3663d3b03e9b 100644 --- a/doc/topics/jobs/index.rst +++ b/doc/topics/jobs/index.rst @@ -20,18 +20,24 @@ cachedir, with a default configuration it is under /var/cache/salt/proc. Functions in the saltutil Module ================================ -Salt 0.9.7 introduced a few new functions to the saltutil module for managing +Salt 0.9.7 introduced a few new functions to the +:doc:`saltutil</ref/modules/all/salt.modules.saltutil>` module for managing jobs. These functions are: -1. running + +1. ``running`` Returns the data of all running jobs that are found in the proc directory. -2. find_job + +2. ``find_job`` Returns specific data about a certain job based on job id. -3. signal_job + +3. ``signal_job`` Allows for a given jid to be sent a signal. -4. term_job + +4. ``term_job`` Sends a termination signal (SIGTERM, 15) to the process controlling the specified job. -5. kill_job + +5. ``kill_job`` Sends a kill signal (SIGKILL, 9) to the process controlling the specified job. From 8567444373058158cb7cfa9ce7eb227d0649ae34 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 18:18:38 -0700 Subject: [PATCH 515/598] War on sphinx war-nings... BRING IT! --- doc/contents.rst | 3 +++ doc/ref/file_server/file_roots.rst | 25 ++++++++++++++----------- doc/topics/installation/index.rst | 2 +- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/doc/contents.rst b/doc/contents.rst index b1be6b281af8..ea46115e5c90 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -13,9 +13,11 @@ Full Table of Contents topics/tutorials/modules topics/tutorials/starting_states topics/tutorials/states* + topics/tutorials/firewall topics/pillar/index topics/jobs/index topics/troubleshooting/index + topics/troubleshooting/yaml_idiosyncrasies topics/community ref/index @@ -32,6 +34,7 @@ Full Table of Contents ref/syndic ref/python-api ref/file_server/index + ref/file_server/dynamic-modules ref/configuration/* ref/cli/index diff --git a/doc/ref/file_server/file_roots.rst b/doc/ref/file_server/file_roots.rst index 1b13d3126830..85418424367e 100644 --- a/doc/ref/file_server/file_roots.rst +++ b/doc/ref/file_server/file_roots.rst @@ -6,10 +6,10 @@ The Salt file server is a high performance file server written in ZeroMQ. It manages large files quickly and with little overhead, and has been optimized to handle small files in an extremely efficient manner. -The Salt file server is an environment aware file server, this means that +The Salt file server is an environment aware file server. This means that files can be allocated within many root directories and accessed by specifying both the file path and the environment to search. The -individual environments can also be spanned across multiple directory roots +individual environments can span across multiple directory roots to crate overlays and to allow for files to be organized in many flexible ways. @@ -17,21 +17,21 @@ Environments ============ The Salt file server defaults to the mandatory ``base`` environment. This -environment MUST be defined and is used to download files when no +environment **MUST** be defined and is used to download files when no environment is specified. Environments allow for files and sls data to be logically separated, but environments are not isolated from each other. This allows for logical isolation of environments by the engineer using Salt, but also allows -for information to be used in multiple environments for maximum flexibility. +for information to be used in multiple environments. Directory Overlay ================= -The environment setting is a list of directories to publish files from. These -directories are searched in order to find the specified file and the first file -found is returned. +The ``environment`` setting is a list of directories to publish files from. +These directories are searched in order to find the specified file and the +first file found is returned. This means that directory data is prioritized based on the order in which they are listed. In the case of this ``file_roots`` configuration: @@ -43,10 +43,10 @@ are listed. In the case of this ``file_roots`` configuration: - /srv/salt/base - /srv/salt/failover -If a file uri os salt://httpd/httpd.conf will first search for the file at -/srv/salt/base/httpd/httpd.conf, if the file is found there it will be -returned, if the file is not found there, then -/srv/salt/failover/httpd/httpd.conf is searched for the file. +If a file's uri is ``salt://httpd/httpd.conf``, it will first search for the +file at ``/srv/salt/base/httpd/httpd.conf``. If the file is found there it +will be returned. If the file is not found there, then +``/srv/salt/failover/httpd/httpd.conf`` will be used for the source. This allows for directories to be overlaid and prioritized based on the order they are defined in the configuration. @@ -54,6 +54,9 @@ they are defined in the configuration. Local File Server ================= +.. versionadded:: 0.9.7 + + The file server can be rerouted to run from the minion. This is primarily to enable running salt states without a salt master. To use the local file server interface, copy the file server data to the minion and set the file_roots diff --git a/doc/topics/installation/index.rst b/doc/topics/installation/index.rst index 3d588f28a739..7324e27635eb 100644 --- a/doc/topics/installation/index.rst +++ b/doc/topics/installation/index.rst @@ -35,7 +35,7 @@ Optional Dependencies .. _`Jinja2`: http://jinja.pocoo.org/ Platform-specific installation instructions ------------------------------------------- +------------------------------------------- .. toctree:: :maxdepth: 1 From 550473093cc2aeb2b7e47e641ac49e59185389e2 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 18:28:03 -0700 Subject: [PATCH 516/598] More missing links added to the table of contents --- doc/contents.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/contents.rst b/doc/contents.rst index ea46115e5c90..f5788dc6c5a7 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -14,6 +14,7 @@ Full Table of Contents topics/tutorials/starting_states topics/tutorials/states* topics/tutorials/firewall + topics/tutorials/bootstrap_ec2 topics/pillar/index topics/jobs/index topics/troubleshooting/index @@ -34,6 +35,7 @@ Full Table of Contents ref/syndic ref/python-api ref/file_server/index + ref/file_server/file_roots ref/file_server/dynamic-modules ref/configuration/* @@ -47,5 +49,6 @@ Full Table of Contents ref/cli/salt-run ref/cli/salt-syndic + topics/specs/salt_auth_proto_abs topics/roadmap/index topics/releases/index From 0b69018ff0e0dc3ebceaf7f35fe92543ac54f653 Mon Sep 17 00:00:00 2001 From: blast_hardcheese <blast@hardchee.se> Date: Sun, 18 Mar 2012 20:14:53 -0700 Subject: [PATCH 517/598] {} in .format() is only valid in 2.7, it seems --- salt/modules/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/apt.py b/salt/modules/apt.py index 825970220fba..bdfefe1db45b 100644 --- a/salt/modules/apt.py +++ b/salt/modules/apt.py @@ -268,7 +268,7 @@ def list_pkgs(regex_string=""): # If ret is empty at this point, check to see if the package is virtual. # We also need aptitude past this point. if not ret and __salt__['cmd.has_exec']('aptitude'): - cmd = ('aptitude search "{} ?virtual ?reverse-provides(?installed)"' + cmd = ('aptitude search "{0} ?virtual ?reverse-provides(?installed)"' .format(regex_string)) out = __salt__['cmd.run_stdout'](cmd) From 0f516bd30d3678ddd56af01b793d1d4502cb42d3 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Sun, 18 Mar 2012 22:17:11 -0600 Subject: [PATCH 518/598] added in 0.9.8 --- doc/ref/file_server/file_roots.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ref/file_server/file_roots.rst b/doc/ref/file_server/file_roots.rst index 85418424367e..13b483601480 100644 --- a/doc/ref/file_server/file_roots.rst +++ b/doc/ref/file_server/file_roots.rst @@ -54,7 +54,7 @@ they are defined in the configuration. Local File Server ================= -.. versionadded:: 0.9.7 +.. versionadded:: 0.9.8 The file server can be rerouted to run from the minion. This is primarily to From 4daef509de3cdf5584c58b8658acefc557b86c99 Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sun, 18 Mar 2012 21:23:17 -0700 Subject: [PATCH 519/598] Always show warnings for duplicate key errors, #631 --- salt/utils/yaml.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/utils/yaml.py b/salt/utils/yaml.py index c08248095b1e..88af64a3676e 100644 --- a/salt/utils/yaml.py +++ b/salt/utils/yaml.py @@ -13,6 +13,13 @@ load = yaml.load + +class DuplicateKeyWarning(RuntimeWarning): + """Warned when duplicate keys exist""" + +warnings.simplefilter('always', category=DuplicateKeyWarning) + + class CustomeConstructor(yaml.constructor.SafeConstructor): def construct_mapping(self, node, deep=False): if not isinstance(node, MappingNode): @@ -29,7 +36,8 @@ def construct_mapping(self, node, deep=False): "found unacceptable key (%s)" % exc, key_node.start_mark) value = self.construct_object(value_node, deep=deep) if key in mapping: - warnings.warn("Duplicate Key: '{0}'".format(key)) + warnings.warn( + "Duplicate Key: '{0}'".format(key), DuplicateKeyWarning) mapping[key] = value return mapping From dac90ddc88ab9a772c9502452abc66598c901db7 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 21:49:30 -0700 Subject: [PATCH 520/598] Updating some of the grammar and facts in the README For instance we use msgpack now, not python pickles. And salt contains a 0mq broker inside of it, not a AMQ broker --- README.rst | 54 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/README.rst b/README.rst index f1f7cc9e5735..b6a57174bef3 100644 --- a/README.rst +++ b/README.rst @@ -11,8 +11,8 @@ Salt is a distributed remote execution system used to execute commands and query data. It was developed in order to bring the best solutions found in the world of remote execution together and make them better, faster and more malleable. Salt accomplishes this via its ability to handle larger loads of -information, and not just dozens, but hundreds or even thousands of individual -servers, handle them quickly and through a simple and manageable interface. +information, and not just dozens, but hundreds, or even thousands of individual +servers. It handles them quickly and through a simple yet manageable interface. Simplicity ========== @@ -30,53 +30,55 @@ Parallel Execution ================== The core function of Salt is to enable remote commands to be called in parallel -rather than in serial, to use a secure and encrypted protocol, the smallest and -fastest network payloads possible, and with a simple programmer interface. Salt -also introduces more granular controls to the realm of remote execution, +rather than in serial. It does this over a secure and encrypted protocol using +the smallest and fastest network payloads possible. All of this is possible +and salt still manages to have a simple interface for developers. Salt also +introduces more granular controls to the realm of remote execution, allowing for commands to be executed in parallel and for systems to be targeted -based on more than just hostname, but by system properties. +based on more than just hostname, but by live system properties. Building on Proven Technology ============================= Salt takes advantage of a number of technologies and techniques. The networking -layer is built with the excellent `ZeroMQ`_ networking library, so Salt itself -contains a viable, and transparent, AMQ broker inside the daemon. Salt uses +layer is built with the excellent `ZeroMQ`_ networking library. Salt itself +contains a viable, and transparent, 0MQ broker inside the daemon. Salt uses public keys for authentication with the master daemon, then uses faster AES -encryption for payload communication, this means that authentication and +encryption for payload communication. This means that authentication and encryption are also built into Salt. Salt takes advantage of communication via -Python pickles, enabling fast and light network traffic. +the most excellent `msgpack`_ library, enabling fast and light network traffic. .. _`ZeroMQ`: http://www.zeromq.org/ +.. _`msgpack`: http://msgpack.org/ + + Python Client Interface ======================= -In order to allow for simple expansion, Salt execution routines can be written -as plain Python modules and the data collected from Salt executions can be sent -back to the master server, or to any arbitrary program. Salt can be called from -a simple Python API, or from the command line, so that Salt can be used to -execute one-off commands as well as operate as an integral part of a larger -application. +Salt execution routines can be written as plain Python modules and the data +collected from execution can be sent back to the master server, or any +arbitrary program. Salt can be called from a simple Python API, or from the +command line. This makes it easy to execute one-off commands as well as +operate as an integral part of a larger application. + Fast, Flexible, Scalable, Secure ================================ The result is a system that can execute commands across groups of varying size, from very few to very many servers at considerably high -speed. A system that is very fast, easy to set up and amazingly -malleable, able to suit the needs of any number of servers working -within the same system. Salt’s unique architecture brings together the -best of the remote execution world, amplifies its capabilities and -expands its range, resulting in this system that is as versatile as it -is practical, able to suit any network. Our source code is pretty -(*pep8*, *pylint*, *pychecker*), well documented, and since we use a +speed. We consider speed to be a feature, not an afterthought. Salt’s +unique architecture brings together the best of the remote execution +world, amplifies its capabilities and expands its range, resulting in +this system that is as versatile as it is practical. Our source code is +pretty (*pep8*, *pylint*, *pychecker*), well documented, and since we use a battle-tested branching model (*gitflow*), we are able to deliver stable software while keeping a fast pace. Last but not least, security is an intrinsic part of salt and something not just influencing how source code is written and how tests are done, but also something that defines the overall architecture and has heavy -influence on the core design of salt. +influence on the core design tenants. Open ==== @@ -84,7 +86,7 @@ Open Salt is developed under the `Apache 2.0 licence`_, and can be used for open and proprietary projects. Please submit your expansions back to the Salt project so that we can all benefit together as Salt grows. -So, please feel free to sprinkle some of this around your systems and -let the deliciousness come forth. +Finally, please sprinkle some salt around your systems and let the +deliciousness come forth. .. _`Apache 2.0 licence`: http://www.apache.org/licenses/LICENSE-2.0.html From d26b89f8f8d8477af8d3cbc9e4fbd9941bc11d5d Mon Sep 17 00:00:00 2001 From: Dan Colish <dcolish@gmail.com> Date: Sun, 18 Mar 2012 21:56:41 -0700 Subject: [PATCH 521/598] Remove Undefined vars and simplify loops in top_merge --- salt/pillar.py | 5 ----- salt/state.py | 5 ----- 2 files changed, 10 deletions(-) diff --git a/salt/pillar.py b/salt/pillar.py index edcb30ac9a08..da251b633adb 100644 --- a/salt/pillar.py +++ b/salt/pillar.py @@ -194,11 +194,6 @@ def merge_tops(self, tops): continue matches = [] states = set() - for comp in ctop[env][tgt]: - if isinstance(comp, dict): - cmatches.append(comp) - if isinstance(comp, basestring): - cstates.add(comp) for comp in top[env][tgt]: if isinstance(comp, dict): matches.append(comp) diff --git a/salt/state.py b/salt/state.py index a71b5c7dcaf4..fe1099868f34 100644 --- a/salt/state.py +++ b/salt/state.py @@ -938,11 +938,6 @@ def merge_tops(self, tops): continue matches = [] states = set() - for comp in ctop[env][tgt]: - if isinstance(comp, dict): - cmatches.append(comp) - if isinstance(comp, basestring): - cstates.add(comp) for comp in top[env][tgt]: if isinstance(comp, dict): matches.append(comp) From 10f1883c6bc7df4fa7525b5d1c68ea9db1e89d73 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Sun, 18 Mar 2012 21:58:56 -0700 Subject: [PATCH 522/598] Update the salt auth abstract to reflect recent changes --- doc/topics/specs/salt_auth_proto_abs.rst | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/topics/specs/salt_auth_proto_abs.rst b/doc/topics/specs/salt_auth_proto_abs.rst index d1540e3c5e16..3c4e737a2667 100644 --- a/doc/topics/specs/salt_auth_proto_abs.rst +++ b/doc/topics/specs/salt_auth_proto_abs.rst @@ -21,7 +21,7 @@ The message itself is abstracted as a python dict in this fashion: .. code-block:: python {'enc': 'aes', - 'load': <encrypted python pickle>} + 'load': <encrypted msgpack binary data>} When this message is received the load can be decrypted using the shared AES key. The 'enc' dict key can also be "pub" for pubkey encryption, or "clear" @@ -90,8 +90,6 @@ Conclusion ========== In the end Salt uses formatted messages with clear header data to specify how -the message data is encrypted. Only uses pubkey encryption for authentication -and to securely retrieve the master AES key. Then all regular communication -is sent in AES encrypted messages. - - +the message data is encrypted. Asymetric encryption via RSA keys is only used +for authentication and to securely retrieve the master AES key. All further +communications are are encrypted with 256 bit AES. From 84b5b5ba497f6ed14b45132c395c8545be853bf9 Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Sun, 18 Mar 2012 19:04:19 -0600 Subject: [PATCH 523/598] Added a couple yaml mock objects for RTD --- doc/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/conf.py b/doc/conf.py index c86a657177e8..cf0f60e09da9 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -30,6 +30,8 @@ def __getattr__(self, name): MOCK_MODULES = [ # salt core 'yaml', + 'yaml.nodes', + 'yaml.constructor', 'msgpack', 'zmq', 'Crypto', From 65fbc41242e07fea6c5d86a5e7d9c1648404aefd Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Sun, 18 Mar 2012 19:05:42 -0600 Subject: [PATCH 524/598] Fixed a couple headline underlines --- doc/ref/modules/all/salt.modules.kvm_hyper.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ref/modules/all/salt.modules.kvm_hyper.rst b/doc/ref/modules/all/salt.modules.kvm_hyper.rst index e54e2e205aeb..63870aee5452 100644 --- a/doc/ref/modules/all/salt.modules.kvm_hyper.rst +++ b/doc/ref/modules/all/salt.modules.kvm_hyper.rst @@ -1,6 +1,6 @@ -================ +====================== salt.modules.kvm_hyper -================ +====================== .. automodule:: salt.modules.kvm_hyper :members: From 867e1ef5cce672004e30a88742c804ad774e450b Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Sun, 18 Mar 2012 19:08:17 -0600 Subject: [PATCH 525/598] rST syntax fixes --- salt/modules/rvm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/rvm.py b/salt/modules/rvm.py index 0f2040de7a5d..d7119fa7af2e 100644 --- a/salt/modules/rvm.py +++ b/salt/modules/rvm.py @@ -228,6 +228,7 @@ def gemset_list_all(runas=None): List all gemsets for all installed rubies. Note that you must have set a default ruby before this can work. + runas : None The user to run rvm as. """ From a7749524a4660c46a76e12209d9408c89a533966 Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Mon, 19 Mar 2012 05:46:20 -0600 Subject: [PATCH 526/598] Fixed old references to grains doc --- doc/topics/tutorials/modules.rst | 2 +- doc/topics/tutorials/states_pt1.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/topics/tutorials/modules.rst b/doc/topics/tutorials/modules.rst index 6ac3f4b6538d..21c1f68844f3 100644 --- a/doc/topics/tutorials/modules.rst +++ b/doc/topics/tutorials/modules.rst @@ -28,7 +28,7 @@ Targets can be based on minion system information using the grains system:: salt -G 'os:Ubuntu' test.ping -.. seealso:: :doc:`Grains system </ref/grains>` +.. seealso:: :doc:`Grains system </topics/targeting/grains>` Targets can be filtered by regular expression:: diff --git a/doc/topics/tutorials/states_pt1.rst b/doc/topics/tutorials/states_pt1.rst index 8fad248a8ed8..04a78699a4d6 100644 --- a/doc/topics/tutorials/states_pt1.rst +++ b/doc/topics/tutorials/states_pt1.rst @@ -60,7 +60,7 @@ minion matches is defined; for now simply specify all hosts (``*``). The expressions can use any of the targeting mechanisms used by Salt — minions can be matched by glob, pcre regular expression, or by :doc:`grains - </ref/grains>`. For example:: + </topics/targeting/grains>`. For example:: base: 'os:Fedora': From 9fdc5a69178d5fb6d35b4b394779e81a11b5055e Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Mon, 19 Mar 2012 05:59:20 -0600 Subject: [PATCH 527/598] Silence autosummary warnings about the virtual modules This change also explicitly shows which modules are virtual. --- doc/ref/modules/all/index.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index 91227d60158e..d1235b09f186 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -4,6 +4,13 @@ Full list of builtin modules ============================ +.. admonition:: Virtual modules + + .. toctree:: + + salt.modules.pkg + salt.modules.sys + .. currentmodule:: salt.modules .. autosummary:: @@ -45,7 +52,6 @@ Full list of builtin modules pacman pillar pip - pkg ps publish puppet @@ -61,7 +67,6 @@ Full list of builtin modules ssh state status - sys systemd test tomcat From 7d1bc6debb2d3df288d57effd30457c5b71421b9 Mon Sep 17 00:00:00 2001 From: Seth House <seth@eseth.com> Date: Mon, 19 Mar 2012 06:07:08 -0600 Subject: [PATCH 528/598] Added link to the perlgeek IRC logs --- doc/topics/community.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/topics/community.rst b/doc/topics/community.rst index 9cff2f656c3b..ca96eeb2ba3c 100644 --- a/doc/topics/community.rst +++ b/doc/topics/community.rst @@ -29,8 +29,11 @@ IRC The ``#salt`` IRC channel is hosted on the popular `Freenode`__ network. You can use the `Freenode webchat client`__ right from your browser. +`Logs of the IRC channel activity`__ are being collected courtesy of Moritz Lenz. + .. __: http://freenode.net/irc_servers.shtml .. __: http://webchat.freenode.net/?channels=salt&uio=Mj10cnVlJjk9dHJ1ZSYxMD10cnVl83 +.. __: http://irclog.perlgeek.de/salt/ .. _community-github: From 5a594b5848689a0a058ed9fe2db255053ca7c669 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 10:25:41 -0600 Subject: [PATCH 529/598] Add run_num verification --- salt/output.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/output.py b/salt/output.py index fad9b64f283a..94f6adc0a0ab 100644 --- a/salt/output.py +++ b/salt/output.py @@ -20,7 +20,6 @@ __all__ = ('get_outputter',) - def display_output(ret, out, opts): ''' Display the output of a command in the terminal @@ -83,8 +82,18 @@ def __call__(self, data, **kwargs): hstrs.append(('{0}----------\n {1}{2[ENDC]}' .format(hcolor, err, colors))) if isinstance(data[host], dict): + # Verify that the needed data is present + for tname, info in data[host].items(): + if not '__run_num__' in info: + err = ('The State execution failed to record the order ' + 'in which all states were executed. The state ' + 'return missing data is:') + print err + pprint.pprint(info) # Everything rendered as it should display the output - for tname in sorted(data[host], key=lambda k: data[host][k]['__run_num__']): + for tname in sorted( + data[host], + key=lambda k: data[host][k].get('__run_num__', 0)): ret = data[host][tname] tcolor = colors['GREEN'] if ret['changes']: From ec9c8e4be8dc82192481d6434995b540b31fd1c9 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 10:37:10 -0600 Subject: [PATCH 530/598] bump version to 0.9.8 --- salt/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/version.py b/salt/version.py index d044332b8eaa..3b2986a236ac 100644 --- a/salt/version.py +++ b/salt/version.py @@ -1,2 +1,2 @@ -__version_info__ = (0, 9, 8, 'pre') +__version_info__ = (0, 9, 8) __version__ = '.'.join(map(str, __version_info__)) From e2d634f06e5b8c180a0f4d290929efac0be8a019 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 10:50:09 -0600 Subject: [PATCH 531/598] fix Cython error --- salt/loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/loader.py b/salt/loader.py index 8f18de672758..6a4ef6c7c129 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -314,8 +314,8 @@ def gen_functions(self, pack=None, virtual_enable=True): pyximport.install() cython_enabled = True except ImportError: - log.info('Cython is enabled in options put not present ' - 'on the system path. Skipping Cython modules.') + log.info('Cython is enabled in the options but not present ' + 'in the system path. Skipping Cython modules.') for mod_dir in self.module_dirs: if not os.path.isabs(mod_dir): continue From dd9b278d2a8e8f22ef8d89e2d6cda47406c470f0 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 11:27:09 -0600 Subject: [PATCH 532/598] only honor enable if all of enable it available --- salt/states/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/service.py b/salt/states/service.py index b3d1d61b34b2..bba711cbbde3 100644 --- a/salt/states/service.py +++ b/salt/states/service.py @@ -47,7 +47,7 @@ def _enable(name, started): 'comment': ''} # Check to see if this minion supports enable - if not 'service.enable' in __salt__: + if not 'service.enable' in __salt__ or not 'service.enabled' in __salt__: if started is True: ret['comment'] = ('Enable is not available on this minion,' ' service {0} started').format(name) @@ -126,7 +126,7 @@ def _disable(name, started): 'comment': ''} # is enable/disable available? - if not 'service.disable' in __salt__: + if not 'service.disable' in __salt__ or not 'service.disabled' in __salt__: if started is True: ret['comment'] = ('Disable is not available on this minion,' ' service {0} started').format(name) From 297968df7466e788261025211deb70f7e1c15c7a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 12:38:23 -0600 Subject: [PATCH 533/598] update salt manpage --- doc/ref/cli/salt.rst | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/doc/ref/cli/salt.rst b/doc/ref/cli/salt.rst index 15c7f7fb0415..5ae1bfab7fd8 100644 --- a/doc/ref/cli/salt.rst +++ b/doc/ref/cli/salt.rst @@ -35,6 +35,20 @@ Options The timeout in seconds to wait for replies from the salt minions. +.. option:: -s STATIC, --static=STATIC + + By default as of version 0.9.8 the salt command returns data to the + console as it is received from minions, but previous releases would return + data only after all data was received. To only return the data with a hard + timeout and after all minions have returned then use the static option. + +.. option:: -b BATCH, --batch-size=BATCH + + Instead of executing on all targeted minions at once, execute on a + progressive set of minions. This option takes an argument in the form of + an explicit number of minions to execute at once, or a percentage of + minions to execute on. + .. option:: --version Print the version of salt that is running. @@ -52,7 +66,17 @@ Options .. option:: -G, --grain The target expression matches values returned by the salt grains system on - the minions. The target expression is in the format of '<grain value>:<pcre + the minions. The target expression is in the format of '<grain value>:<glob + expression>'; example: 'os:Arch*' + + This was changed in version 0.9.8 to accept glob expressions instead of + regular expression. To use regular expression matching with grains use + the --grain-pcre option. + +.. option:: --grain-pcre + + The target expression matches values returned by the salt grains system on + the minions. The target expression is in the format of '<grain value>:< regular expression>'; example: 'os:Arch.*' .. option:: -C, --compound @@ -60,7 +84,7 @@ Options Utilize many target definitions to make the call very granular. This option takes a group of targets separated by and or or. The default matcher is a glob as usual, if something other than a glob is used preface it with the - letter denoting the type, example: 'webserv* and G@os:Debian or E@db.*' + letter denoting the type, example: 'webserv* and G@os:Debian or E@db*' make sure that the compound target is encapsulated in quotes. .. option:: -X, --exsel @@ -72,6 +96,15 @@ Options Use a predefined compound target defined in the salt master configuration file +.. option:: -R, --range + + Instead of using shell globs to evaluate the targe use a range expression + to identify targets. Range expressions look like %cluster. + + Using the Range option requires that a range server is set up and the + location of the range server is referenced in the master configuration + file. + .. option:: --return Chose an alternative returner to call on the minion, if an alternative @@ -80,7 +113,7 @@ Options .. option:: -Q, --query - The -Q option is being deprecated and will be removed in a future release, + The -Q option is being deprecated and will be removed in version 0.9.9, Use the salt jobs interface instead, for documentation on the salt jobs interface execute the command "salt-run -d jobs" @@ -112,6 +145,10 @@ Options Print the output from the salt command in json. +.. option:: --no-color + + Disable all colored output + See also ======== From eda3377da7cb29cfa150e957e046aeb2de4e97c9 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 12:41:21 -0600 Subject: [PATCH 534/598] Add new components to salt-cp manpage --- doc/ref/cli/salt-cp.rst | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/doc/ref/cli/salt-cp.rst b/doc/ref/cli/salt-cp.rst index a0e29fbeefd5..6f47a2316bca 100644 --- a/doc/ref/cli/salt-cp.rst +++ b/doc/ref/cli/salt-cp.rst @@ -46,14 +46,32 @@ Options .. option:: -G, --grain + The target expression matches values returned by the salt grains system on + the minions. The target expression is in the format of '<grain value>:<glob + expression>'; example: 'os:Arch*' + +.. option:: --grain-pcre + The target expression matches values returned by the salt grains system on the minions. The target expression is in the format of '<grain value>:<pcre regular expression>'; example: 'os:Arch.*' -.. option:: -Q, --query +.. option:: -R, --range + + Instead of using shell globs to evaluate the targe use a range expression + to identify targets. Range expressions look like %cluster. + + Using the Range option requires that a range server is set up and the + location of the range server is referenced in the master configuration + file. + +.. option:: -C, --compound - Execute a salt command query, this can be used to find the results of a - previous function call: -Q test.echo') + Utilize many target definitions to make the call very granular. This option + takes a group of targets separated by and or or. The default matcher is a + glob as usual, if something other than a glob is used preface it with the + letter denoting the type, example: 'webserv* and G@os:Debian or E@db*' + make sure that the compound target is encapsulated in quotes. .. option:: -c CONFIG, --config=CONFIG From 66f2674f8b1abf89ca57175c4444f7b8f8ed2a40 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 12:44:24 -0600 Subject: [PATCH 535/598] update salt-key manpage --- doc/ref/cli/salt-key.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/ref/cli/salt-key.rst b/doc/ref/cli/salt-key.rst index bab9463ddb68..16bf42095000 100644 --- a/doc/ref/cli/salt-key.rst +++ b/doc/ref/cli/salt-key.rst @@ -47,10 +47,14 @@ Options Rejects all pending public keys. - option:: -d DELETE --delete=DELETE +.. option:: -d DELETE, --delete=DELETE Delete the named minion key for command execution. +.. option:: -D DELETE_ALL, --delete-all=DELETE_ALL + + Deleta all keys + .. option:: -c CONFIG, --config=CONFIG The master configuration file needs to be read to determine where the salt From 91ecf8e55047ab9f13c8e0ef24052d7365dc4f8e Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 12:47:05 -0600 Subject: [PATCH 536/598] update manpage salt-call --- doc/ref/cli/salt-call.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/doc/ref/cli/salt-call.rst b/doc/ref/cli/salt-call.rst index 646807ac182d..4f94123fa7b1 100644 --- a/doc/ref/cli/salt-call.rst +++ b/doc/ref/cli/salt-call.rst @@ -31,3 +31,32 @@ Options Return the documentation for the specified module of for all modules if none are specified + +.. option:: -l LOG_LEVEL, --log-level=LOG_LEVEL + + Console log level. One of ``info``, ``none``, ``garbage``, + ``trace``, ``warning``, ``error``, ``debug``. For the logfile + settings see the config file. Default: ``info``. + +.. option:: --raw-out + + Print the output from the salt command in raw python + form, this is suitable for re-reading the output into + an executing python script with eval. + +.. option:: --text-out + + Print the output from the salt command in the same + form the shell would. + +.. option:: --yaml-out + + Print the output from the salt command in yaml. + +.. option:: --json-out + + Print the output from the salt command in json. + +.. option:: --no-color + + Disable all colored output From efca4be0490cac47c100af607793c3b1bc95cf90 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 12:54:49 -0600 Subject: [PATCH 537/598] update salt-master manpage --- doc/ref/cli/salt-master.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/ref/cli/salt-master.rst b/doc/ref/cli/salt-master.rst index f75ee4e80638..964054a3c602 100644 --- a/doc/ref/cli/salt-master.rst +++ b/doc/ref/cli/salt-master.rst @@ -35,6 +35,10 @@ Options Specify user to run minion +.. option:: --pid-file PIDFILE + + Specify the location of the pidfile. + .. option:: -l LOG_LEVEL, --log-level=LOG_LEVEL Console log level. One of ``info``, ``none``, ``garbage``, From bf6117cb530fa4e1876335e5b3f8f589b1a3790e Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 12:57:46 -0600 Subject: [PATCH 538/598] update salt-minion manpage --- doc/ref/cli/salt-minion.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/ref/cli/salt-minion.rst b/doc/ref/cli/salt-minion.rst index 7254566e5d45..7b608058360c 100644 --- a/doc/ref/cli/salt-minion.rst +++ b/doc/ref/cli/salt-minion.rst @@ -36,6 +36,10 @@ Options Specify user to run minion +.. option:: --pid-file PIDFILE + + Specify the location of the pidfile + .. option:: -l LOG_LEVEL, --log-level=LOG_LEVEL Console log level. One of ``info``, ``none``, ``garbage``, From 66cb8daba7c5a884bececb2151fac663db007737 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 12:58:15 -0600 Subject: [PATCH 539/598] Update salt-syndic manpage --- doc/ref/cli/salt-syndic.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/ref/cli/salt-syndic.rst b/doc/ref/cli/salt-syndic.rst index ed615c3c3604..e5511ee5cb95 100644 --- a/doc/ref/cli/salt-syndic.rst +++ b/doc/ref/cli/salt-syndic.rst @@ -29,6 +29,10 @@ Options Run the salt syndic as a daemon +.. option:: --pid-file PIDFILE + + Specify the location of the pidfile + .. option:: --master-config=MASTER_CONFIG The master configuration file to use, the default is /etc/salt/master From a8f48832af03aeb83899173d77380ed29e9aa80d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 13:02:36 -0600 Subject: [PATCH 540/598] Update live manpages for 0.9.8 --- doc/man/salt-call.1 | 37 +- doc/man/salt-cp.1 | 28 +- doc/man/salt-key.1 | 12 +- doc/man/salt-master.1 | 7 +- doc/man/salt-minion.1 | 7 +- doc/man/salt-run.1 | 2 +- doc/man/salt-syndic.1 | 7 +- doc/man/salt.1 | 52 +- doc/man/salt.7 | 13133 +++++++++++++++++++++++++++++----------- 9 files changed, 9712 insertions(+), 3573 deletions(-) diff --git a/doc/man/salt-call.1 b/doc/man/salt-call.1 index 58f5d91bf33e..e5e9d9ed77e1 100644 --- a/doc/man/salt-call.1 +++ b/doc/man/salt-call.1 @@ -1,4 +1,4 @@ -.TH "SALT-CALL" "1" "February 15, 2012" "0.9.7" "Salt" +.TH "SALT-CALL" "1" "March 19, 2012" "0.9.8" "Salt" .SH NAME salt-call \- salt-call Documentation . @@ -60,6 +60,41 @@ directories can be delimited by commas Return the documentation for the specified module of for all modules if none are specified .UNINDENT +.INDENT 0.0 +.TP +.B \-l LOG_LEVEL, \-\-log\-level=LOG_LEVEL +Console log level. One of \fBinfo\fP, \fBnone\fP, \fBgarbage\fP, +\fBtrace\fP, \fBwarning\fP, \fBerror\fP, \fBdebug\fP. For the logfile +settings see the config file. Default: \fBinfo\fP. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-raw\-out +Print the output from the salt command in raw python +form, this is suitable for re\-reading the output into +an executing python script with eval. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-text\-out +Print the output from the salt command in the same +form the shell would. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-yaml\-out +Print the output from the salt command in yaml. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-json\-out +Print the output from the salt command in json. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-color +Disable all colored output +.UNINDENT .SH AUTHOR Thomas S. Hatch <thatch@gmail.com> and many others, please see the Authors file .SH COPYRIGHT diff --git a/doc/man/salt-cp.1 b/doc/man/salt-cp.1 index 3e8f8e2476bb..a50188b3cb65 100644 --- a/doc/man/salt-cp.1 +++ b/doc/man/salt-cp.1 @@ -1,4 +1,4 @@ -.TH "SALT-CP" "1" "February 15, 2012" "0.9.7" "Salt" +.TH "SALT-CP" "1" "March 19, 2012" "0.9.8" "Salt" .SH NAME salt-cp \- salt-cp Documentation . @@ -74,14 +74,34 @@ example: server1.foo.bar,server2.foo.bar,example7.quo.qux .TP .B \-G, \-\-grain The target expression matches values returned by the salt grains system on +the minions. The target expression is in the format of \(aq<grain value>:<glob +expression>\(aq; example: \(aqos:Arch*\(aq +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-grain\-pcre +The target expression matches values returned by the salt grains system on the minions. The target expression is in the format of \(aq<grain value>:<pcre regular expression>\(aq; example: \(aqos:Arch.*\(aq .UNINDENT .INDENT 0.0 .TP -.B \-Q, \-\-query -Execute a salt command query, this can be used to find the results of a -previous function call: \-Q test.echo\(aq) +.B \-R, \-\-range +Instead of using shell globs to evaluate the targe use a range expression +to identify targets. Range expressions look like %cluster. +.sp +Using the Range option requires that a range server is set up and the +location of the range server is referenced in the master configuration +file. +.UNINDENT +.INDENT 0.0 +.TP +.B \-C, \-\-compound +Utilize many target definitions to make the call very granular. This option +takes a group of targets separated by and or or. The default matcher is a +glob as usual, if something other than a glob is used preface it with the +letter denoting the type, example: \(aqwebserv* and \fI\%G@os\fP:Debian or \fI\%E@db*\fP\(aq +make sure that the compound target is encapsulated in quotes. .UNINDENT .INDENT 0.0 .TP diff --git a/doc/man/salt-key.1 b/doc/man/salt-key.1 index 1892e1f2fa88..5f05e333e672 100644 --- a/doc/man/salt-key.1 +++ b/doc/man/salt-key.1 @@ -1,4 +1,4 @@ -.TH "SALT-KEY" "1" "February 15, 2012" "0.9.7" "Salt" +.TH "SALT-KEY" "1" "March 19, 2012" "0.9.8" "Salt" .SH NAME salt-key \- salt-key Documentation . @@ -73,10 +73,16 @@ Reject the named minion public key. .TP .B \-R, \-\-reject\-all Rejects all pending public keys. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d DELETE, \-\-delete=DELETE +Delete the named minion key for command execution. +.UNINDENT .INDENT 0.0 .TP -.B \-d, \-\-delete=DELETE -Delete the named minion public key for command execution. +.B \-D DELETE_ALL, \-\-delete\-all=DELETE_ALL +Deleta all keys .UNINDENT .INDENT 0.0 .TP diff --git a/doc/man/salt-master.1 b/doc/man/salt-master.1 index a04afbc3db86..a62f1689a8c3 100644 --- a/doc/man/salt-master.1 +++ b/doc/man/salt-master.1 @@ -1,4 +1,4 @@ -.TH "SALT-MASTER" "1" "February 15, 2012" "0.9.7" "Salt" +.TH "SALT-MASTER" "1" "March 19, 2012" "0.9.8" "Salt" .SH NAME salt-master \- salt-master Documentation . @@ -61,6 +61,11 @@ Specify user to run minion .UNINDENT .INDENT 0.0 .TP +.B \-\-pid\-file PIDFILE +Specify the location of the pidfile. +.UNINDENT +.INDENT 0.0 +.TP .B \-l LOG_LEVEL, \-\-log\-level=LOG_LEVEL Console log level. One of \fBinfo\fP, \fBnone\fP, \fBgarbage\fP, \fBtrace\fP, \fBwarning\fP, \fBerror\fP, \fBdebug\fP. For the logfile diff --git a/doc/man/salt-minion.1 b/doc/man/salt-minion.1 index 3d0a653519ce..1c5ac702fb53 100644 --- a/doc/man/salt-minion.1 +++ b/doc/man/salt-minion.1 @@ -1,4 +1,4 @@ -.TH "SALT-MINION" "1" "February 15, 2012" "0.9.7" "Salt" +.TH "SALT-MINION" "1" "March 19, 2012" "0.9.8" "Salt" .SH NAME salt-minion \- salt-minion Documentation . @@ -62,6 +62,11 @@ Specify user to run minion .UNINDENT .INDENT 0.0 .TP +.B \-\-pid\-file PIDFILE +Specify the location of the pidfile +.UNINDENT +.INDENT 0.0 +.TP .B \-l LOG_LEVEL, \-\-log\-level=LOG_LEVEL Console log level. One of \fBinfo\fP, \fBnone\fP, \fBgarbage\fP, \fBtrace\fP, \fBwarning\fP, \fBerror\fP, \fBdebug\fP. For the logfile diff --git a/doc/man/salt-run.1 b/doc/man/salt-run.1 index 0248f8ff0116..ad043f883be1 100644 --- a/doc/man/salt-run.1 +++ b/doc/man/salt-run.1 @@ -1,4 +1,4 @@ -.TH "SALT-RUN" "1" "February 15, 2012" "0.9.7" "Salt" +.TH "SALT-RUN" "1" "March 19, 2012" "0.9.8" "Salt" .SH NAME salt-run \- salt-run Documentation . diff --git a/doc/man/salt-syndic.1 b/doc/man/salt-syndic.1 index a422c6a0dcd0..63d514c1d00c 100644 --- a/doc/man/salt-syndic.1 +++ b/doc/man/salt-syndic.1 @@ -1,4 +1,4 @@ -.TH "SALT-SYNDIC" "1" "February 15, 2012" "0.9.7" "Salt" +.TH "SALT-SYNDIC" "1" "March 19, 2012" "0.9.8" "Salt" .SH NAME salt-syndic \- salt-syndic Documentation . @@ -53,6 +53,11 @@ Run the salt syndic as a daemon .UNINDENT .INDENT 0.0 .TP +.B \-\-pid\-file PIDFILE +Specify the location of the pidfile +.UNINDENT +.INDENT 0.0 +.TP .B \-\-master\-config=MASTER_CONFIG The master configuration file to use, the default is /etc/salt/master .UNINDENT diff --git a/doc/man/salt.1 b/doc/man/salt.1 index 456485adf249..89f6216dae25 100644 --- a/doc/man/salt.1 +++ b/doc/man/salt.1 @@ -1,4 +1,4 @@ -.TH "SALT" "1" "February 15, 2012" "0.9.7" "Salt" +.TH "SALT" "1" "March 19, 2012" "0.9.8" "Salt" .SH NAME salt \- salt . @@ -62,6 +62,22 @@ The timeout in seconds to wait for replies from the salt minions. .UNINDENT .INDENT 0.0 .TP +.B \-s STATIC, \-\-static=STATIC +By default as of version 0.9.8 the salt command returns data to the +console as it is received from minions, but previous releases would return +data only after all data was received. To only return the data with a hard +timeout and after all minions have returned then use the static option. +.UNINDENT +.INDENT 0.0 +.TP +.B \-b BATCH, \-\-batch\-size=BATCH +Instead of executing on all targeted minions at once, execute on a +progressive set of minions. This option takes an argument in the form of +an explicit number of minions to execute at once, or a percentage of +minions to execute on. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-version Print the version of salt that is running. .UNINDENT @@ -81,7 +97,18 @@ example: server1.foo.bar,server2.foo.bar,example7.quo.qux .TP .B \-G, \-\-grain The target expression matches values returned by the salt grains system on -the minions. The target expression is in the format of \(aq<grain value>:<pcre +the minions. The target expression is in the format of \(aq<grain value>:<glob +expression>\(aq; example: \(aqos:Arch*\(aq +.sp +This was changed in version 0.9.8 to accept glob expressions instead of +regular expression. To use regular expression matching with grains use +the \-\-grain\-pcre option. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-grain\-pcre +The target expression matches values returned by the salt grains system on +the minions. The target expression is in the format of \(aq<grain value>:< regular expression>\(aq; example: \(aqos:Arch.*\(aq .UNINDENT .INDENT 0.0 @@ -90,7 +117,7 @@ regular expression>\(aq; example: \(aqos:Arch.*\(aq Utilize many target definitions to make the call very granular. This option takes a group of targets separated by and or or. The default matcher is a glob as usual, if something other than a glob is used preface it with the -letter denoting the type, example: \(aqwebserv* and \fI\%G@os\fP:Debian or \fI\%E@db.*\fP\(aq +letter denoting the type, example: \(aqwebserv* and \fI\%G@os\fP:Debian or \fI\%E@db*\fP\(aq make sure that the compound target is encapsulated in quotes. .UNINDENT .INDENT 0.0 @@ -106,6 +133,16 @@ file .UNINDENT .INDENT 0.0 .TP +.B \-R, \-\-range +Instead of using shell globs to evaluate the targe use a range expression +to identify targets. Range expressions look like %cluster. +.sp +Using the Range option requires that a range server is set up and the +location of the range server is referenced in the master configuration +file. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-return Chose an alternative returner to call on the minion, if an alternative returner is used then the return will not come back tot he command line @@ -114,6 +151,10 @@ but will be sent to the specified return system. .INDENT 0.0 .TP .B \-Q, \-\-query +The \-Q option is being deprecated and will be removed in version 0.9.9, +Use the salt jobs interface instead, for documentation on the salt jobs +interface execute the command "salt\-run \-d jobs" +.sp Execute a salt command query, this can be used to find the results of a previous function call: \-Q test.echo\(aq) .UNINDENT @@ -147,6 +188,11 @@ Print the output from the salt command in yaml. .B \-\-json\-out Print the output from the salt command in json. .UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-color +Disable all colored output +.UNINDENT .SH SEE ALSO .sp \fIsalt(7)\fP diff --git a/doc/man/salt.7 b/doc/man/salt.7 index f306e5154786..42ee9d282245 100644 --- a/doc/man/salt.7 +++ b/doc/man/salt.7 @@ -1,4 +1,4 @@ -.TH "SALT" "7" "February 15, 2012" "0.9.7" "Salt" +.TH "SALT" "7" "March 19, 2012" "0.9.8" "Salt" .SH NAME salt \- Salt Documentation . @@ -89,1357 +89,3537 @@ Salt is developed under the \fI\%Apache 2.0 licence\fP, and can be used for open proprietary projects. Please submit your expansions back to the Salt project so that we can all benefit together as Salt grows. So, please feel free to sprinkle some of this around your systems and let the deliciousness come forth. -.SH CONFIGURING SALT +.SH INSTALLATION .sp -Salt configuration is very simple. The default configuration for the -\fImaster\fP will work for most installations and the only requirement for -setting up a \fIminion\fP is to set the location of the master in the minion -configuration file. +The Salt system setup is amazingly simple, as this is one of the central design +goals of Salt. +.SS Dependencies +.sp +Salt should run on any Unix\-like platform so long as the dependencies are met. .INDENT 0.0 -.TP -.B master -The Salt master is the central server that all minions connect to. You -run commands on the minions through the master and minions send data -back to the master (unless otherwise redirected with a \fBreturner\fP). It is started with the -\fBsalt\-master\fP program. -.TP -.B minion -Salt minions are the potentially hundreds or thousands of servers that -you query and control from the master. +.IP \(bu 2 +\fI\%Python 2.6\fP +.IP \(bu 2 +\fI\%ZeroMQ\fP >= 2.1.9 +.IP \(bu 2 +\fI\%pyzmq\fP >= 2.1.9 \- ZeroMQ Python bindings +.IP \(bu 2 +\fI\%M2Crypto\fP \- Python OpenSSL wrapper +.IP \(bu 2 +\fI\%PyCrypto\fP \- The Python cryptography toolkit +.IP \(bu 2 +\fI\%msgpack-python\fP \- High\-performance message interchange format +.IP \(bu 2 +\fI\%YAML\fP \- Python YAML bindings +.UNINDENT +.SS Optional Dependencies +.INDENT 0.0 +.IP \(bu 2 +\fI\%Jinja2\fP \- parsing Salt States (configurable in the master settings) +.IP \(bu 2 +gcc \- dynamic \fI\%Cython\fP module compiling .UNINDENT +.SS Platform\-specific installation instructions +.SS Arch Linux .sp -The configuration files will be installed to \fB/etc/salt\fP and are named -after the respective components, \fB/etc/salt/master\fP and -\fB/etc/salt/minion\fP. +Salt has primarily been developed on Arch Linux, meaning it is known to +work very well on that distribution. The lead developer, Thomas S. Hatch +(thatch45) has been a TU (Trusted User) for the Arch Linux distribution, +and has written a number of Arch\-specific tools in the past. .sp -To make a minion check into the correct master simply edit the -\fBmaster\fP variable in the minion configuration file to reference -the master DNS name or IPv4 address. -.SS Running Salt +Salt, while not Arch\-specific, is packaged for and works well on Arch Linux. +.SS Installation +.sp +Salt is currently available via the Arch User Repository (AUR). There are +currently stable and \-git packages available. +.SS Stable Release +.sp +To install Salt stable releases from the Arch Linux AUR, use the commands: +.sp +.nf +.ft C +wget https://aur.archlinux.org/packages/sa/salt/salt.tar.gz +tar xf salt.tar.gz +cd salt/ +makepkg \-is +.ft P +.fi +.sp +A few of Salt\(aqs dependencies are currently only found within the AUR, so you\(aqll +need to download and run \fBmakepkg \-is\fP on these as well. As a reference, Salt +currently relies on the following packages only available via the AUR: .INDENT 0.0 -.IP 1. 3 -Start the master in the foreground (to daemonize the process, pass the -\fI\-d flag\fP): +.IP \(bu 2 +\fI\%https://aur.archlinux.org/packages/py/python2-msgpack/python2-msgpack.tar.gz\fP +.IP \(bu 2 +\fI\%https://aur.archlinux.org/packages/py/python2-psutil/python2-psutil.tar.gz\fP +.UNINDENT +.IP Note +yaourt +.sp +If you chose to use a tool such as \fI\%Yaourt\fP the dependencies will be +gathered and built for you automatically. +.sp +The command to install salt using the yaourt tool is: .sp .nf .ft C -# salt\-master +yaourt salt .ft P .fi -.IP 2. 3 -Start the minion in the foreground (to daemonize the process, pass the -\fI\-d flag\fP): +.RE +.SS Tracking develop +.sp +To install the bleeding edge version of Salt (\fBmay include bugs!\fP), +you can use the \-git package. Installing the \-git package can be done +using the commands: .sp .nf .ft C -# salt\-minion +wget https://aur.archlinux.org/packages/sa/salt\-git/salt\-git.tar.gz +tar xf salt\-git.tar.gz +cd salt\-git/ +makepkg \-is .ft P .fi +.sp +A few of Salt\(aqs dependencies are currently only found within the AUR, so you\(aqll +need to download and run \fBmakepkg \-is\fP on these as well. As a reference, Salt +currently relies on the following packages only available via the AUR: +.INDENT 0.0 +.IP \(bu 2 +\fI\%https://aur.archlinux.org/packages/py/python2-msgpack/python2-msgpack.tar.gz\fP +.IP \(bu 2 +\fI\%https://aur.archlinux.org/packages/py/python2-psutil/python2-psutil.tar.gz\fP .UNINDENT -.IP "Having trouble?" +.IP Note +yaourt .sp -The simplest way to troubleshoot Salt is to run the master and minion in -the foreground with \fIlog level\fP set to \fBdebug\fP: +If you chose to use a tool such as \fI\%Yaourt\fP the dependencies will be +gathered and built for you automatically. +.sp +The command to install salt using the yaourt tool is: .sp .nf .ft C -salt\-master \-\-log\-level=debug +yaourt salt\-git .ft P .fi .RE -.IP "Run as an unprivileged (non\-root) user?" +.SS Configuration .sp -To run Salt as another user, specify \fB\-\-user\fP in the command -line or assign \fBuser\fP in the -\fBconfiguration file\fP. -.RE -.SS Manage Salt public keys +In the sections below I\(aqll outline configuration options for both the Salt +Master and Salt Minions. .sp -Salt manages authentication with RSA public keys. The keys are managed on the -\fImaster\fP via the \fBsalt\-key\fP command. Once a \fIminion\fP -checks into the master the master will save a copy of the minion key. Before -the master can send commands to the minion the key needs to be "accepted". -.INDENT 0.0 -.IP 1. 3 -List the accepted and unaccepted salt keys: +The Salt package installs two template configuration files, +\fB/etc/salt/master.template\fP and \fB/etc/salt/minion.template\fP. You\(aqll need +to copy these .template files into place and make a few edits. First, copy +them into place as seen here: .sp .nf .ft C -salt\-key \-L +cp /etc/salt/master.template /etc/salt/master +cp /etc/salt/minion.template /etc/salt/minion .ft P .fi -.IP 2. 3 -Accept a minion key: +.sp +Note: You\(aqll only need to copy the config for the service you\(aqre going to run. +.sp +Once you\(aqve copied the config into place you\(aqll need to make changes specific +to your setup. Below I\(aqll outline suggested configuration changes to the +Master, after which I\(aqll outline configuring the Minion. +.SS Master Configuration +.sp +This section outlines configuration of a Salt Master, which is used to control +other machines known as "minions" (see "Minion Configuration" for instructions +on configuring a minion). This will outline IP configuration, and a few key +configuration paths. +.sp +\fBInterface\fP +.sp +By default the Salt master listens on ports 4505 and 4506 on all interfaces +(0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the +"interface" directive as seen here: .sp .nf .ft C -salt\-key \-a <minion id> +\- #interface: 0.0.0.0 ++ interface: 10.0.0.1 .ft P .fi .sp -or accept all unaccepted minion keys: +\fBrc.conf\fP +.sp +You\(aqll need to activate the Salt Master in your rc.conf file. Using your +favorite editor, open \fB/etc/rc.conf\fP and add the salt\-master. .sp .nf .ft C -salt\-key \-A +\-DAEMONS=(syslog\-ng network crond) ++DAEMONS=(syslog\-ng network crond @salt\-master) .ft P .fi -.UNINDENT -.IP "See also" .sp -\fBsalt\-key manpage\fP -.RE -.SH REMOTE EXECUTION TUTORIAL +\fBStart the Master\fP .sp -\fBBefore continuing\fP make sure you have a working Salt installation by -following the \fI\%installation\fP and the \fBconfiguration\fP instructions. -.IP "Stuck?" +Once you\(aqve completed all of these steps you\(aqre ready to start your Salt +Master. You should be able to start your Salt Master now using the command +seen here: .sp -If you get stuck at any point, there are many ways to \fBget help from -the Salt community\fP including our mailing list and our -IRC channel. -.RE -.SS Order your minions around +.nf +.ft C +rc.d start salt\-master +.ft P +.fi .sp -Now that you have a \fImaster\fP and at least one \fIminion\fP -communicating with each other you can perform commands on the minion via the -\fBsalt\fP command. Salt calls are comprised of three main components: +If your Salt Master doesn\(aqt start successfully, go back through each step and +see if anything was missed. Salt doesn\(aqt take much configuration (part of its +beauty!), and errors are usually simple mistakes. +.SS Minion Configuration +.sp +Configuring a Salt Minion is surprisingly simple. Unless you have a real need +for customizing your minion configuration (which there are plenty of options if +you are so inclined!), there is one simple directive that needs to be updated. +That option is the location of the master. +.sp +By default a Salt Minion will try to connect to the dns name "salt". If you +have the ability to update DNS records for your domain you might create an A or +CNAME record for "salt" that points to your Salt Master. If you are able to do +this you likely can do without any minion configuration at all. +.sp +If you are not able to update DNS, you\(aqll simply need to update one entry in +the configuration file. Using your favorite editor, open the minion +configuration file and update the "master" entry as seen here. .sp .nf .ft C -salt \(aq<target>\(aq <function> [arguments] +\- #master: salt ++ master: 10.0.0.1 .ft P .fi -.IP "See also" .sp -\fBsalt manpage\fP -.RE -.SS target +Simply update the master directive to the IP or hostname of your Salt Master. +Save your changes and you\(aqre ready to start your Salt Minion. Advanced +configuration options are covered in another chapter. .sp -The target component allows you to filter which minions should run the -following function. The default filter is a glob on the minion id. E.g.: +\fBrc.conf\fP +.sp +Before you\(aqre able to start the Salt Minion you\(aqll need to update your rc.conf +file. Using your favorite editor open \fB/etc/rc.conf\fP and add this line: .sp .nf .ft C -salt \(aq*\(aq test.ping -salt \(aq*.example.org\(aq test.ping +\-DAEMONS=(syslog\-ng network crond) ++DAEMONS=(syslog\-ng network crond @salt\-minion) .ft P .fi .sp -Targets can be based on minion system information using the grains system: +\fBStart the Minion\fP +.sp +Once you\(aqve completed all of these steps you\(aqre ready to start your Salt +Minion. You should be able to start your Salt Minion now using the command +seen here: .sp .nf .ft C -salt \-G \(aqos:Ubuntu\(aq test.ping +rc.d start salt\-minion .ft P .fi -.IP "See also" .sp -\fBGrains system\fP -.RE +If your Salt Minion doesn\(aqt start successfully, go back through each step and +see if anything was missed. Salt doesn\(aqt take much configuration (part of its +beauty!), and errors are usually simple mistakes. +.SS Tying It All Together .sp -Targets can be filtered by regular expression: +If you\(aqve successfully completed each of the steps above you should have a +running Salt Master and a running Salt Minion. The Minion should be configured +to point to the Master. To verify that there is communication flowing between +the Minion and Master we\(aqll run a few initial \fBsalt\fP commands. These commands +will validate the Minions RSA encryption key, and then send a test command to +the Minion to ensure that commands and responses are flowing as expected. +.sp +\fBKey Management\fP +.sp +Salt uses AES encryption for all communication between the Master and the +Minion. This ensures that the commands you send to your Minions (your cloud) +can not be tampered with, and that communication between Master and Minion is +only done through trusted, accepted keys. +.sp +Before you\(aqll be able to do any remote execution or configuration management you\(aqll +need to accept any pending keys on the Master. Run the \fBsalt\-key\fP command to +list the keys known to the Salt Master. .sp .nf .ft C -salt \-E \(aqvirtmach[0\-9]\(aq test.ping +[root@master ~]# salt\-key \-L +Unaccepted Keys: +alpha +bravo +charlie +delta +Accepted Keys: .ft P .fi .sp -Finally, targets can be explicitly specified in a list: +This example shows that the Salt Master is aware of four Minions, but none of +the keys have been accepted. To accept the keys and allow the Minions to be +controlled by the Master, again use the \fBsalt\-key\fP command: .sp .nf .ft C -salt \-L foo,bar,baz,quo test.ping +[root@master ~]# salt\-key \-A +[root@master ~]# salt\-key \-L +Unaccepted Keys: +Accepted Keys: +alpha +bravo +charlie +delta .ft P .fi -.SS function .sp -A function is some functionality provided by a module. Salt ships with a large -collection of available functions. List all available functions on your -minions: +The \fBsalt\-key\fP command allows for signing keys individually or in bulk. The +example above, using \fB\-A\fP bulk\-accepts all pending keys. To accept keys +individually use the lowercase of the same option, \fB\-a keyname\fP. +.SS Sending Commands +.sp +Everything should be set for you to begin remote management of your Minions. +Whether you have a few or a few\-dozen, Salt can help you manage them easily! +.sp +For final verification, send a test function from your Salt Master to your +minions. If all of your minions are properly communicating with your Master, +you should "True" responses from each of them. See the example below to send +the \fBtest.ping\fP remote command: .sp .nf .ft C -salt \(aq*\(aq sys.doc +[root@master ~]# salt \(aq*\(aq test.ping +{\(aqalpha\(aq: True} .ft P .fi +.SS Where Do I Go From Here .sp -Here are some examples: +Congratulations! You\(aqve successfully configured your first Salt Minions and are +able to send remote commands. I\(aqm sure you\(aqre eager to learn more about what +Salt can do. Depending on the primary way you want to manage your machines you +may either want to visit the section regarding Salt States, or the section on +Modules. +.SS Debian & Ubuntu +.SS Ubuntu .sp -Show all currently available minions: +We are working to get Salt into apt. In the meantime we have a PPA available +for Lucid: .sp .nf .ft C -salt \(aq*\(aq test.ping +aptitude \-y install python\-software\-properties +add\-apt\-repository ppa:saltstack/salt +aptitude update +aptitude install salt\-master # on the master +aptitude install salt\-minion # on the minion +aptitude install salt\-syndic # instead of a slaved master .ft P .fi +.SS Debian .sp -Run an arbitrary shell command: +\fI\%A deb package is currently in testing\fP for inclusion in apt. Until that is +accepted you can install Salt by downloading the latest \fB.deb\fP in the +\fI\%downloads section on GitHub\fP and installing that manually using \fBdpkg \-i\fP. +.IP "Installing ZeroMQ on Squeeze (Debian 6)" +.sp +There is a \fI\%python-zmq\fP package available in Debian "wheezy (testing)". +If you don\(aqt have that repo enabled the best way to install Salt and pyzmq +is by using \fBpip\fP (or \fBeasy_install\fP): .sp .nf .ft C -salt \(aq*\(aq cmd.run \(aquname \-a\(aq +pip install pyzmq salt .ft P .fi -.IP "See also" -.sp -\fBthe full list of modules\fP .RE -.SS arguments +.SS Fedora & CentOS / Enterprise Linux .sp -Space\-delimited arguments to the function: +Beginning with version 0.9.4, Salt has been available in the primary Fedora +repositories and is available for installation using yum. Fedora will have more +up to date versions of Salt than other members of the Red Hat family, which +makes it a great place to help improve Salt! +.IP "CentOS/EL 5" +.sp +We are still working to get msgpack packaged for Python 2.6 so Salt is not +yet (but nearly) in EPEL 5. In the meantime you can install Salt via our +Fedora People repository: .sp .nf .ft C -salt \(aq*\(aq cmd.exec_code python \(aqimport sys; print sys.version\(aq +wget \-O /etc/yum.repos.d/epel\-salt.repo \e\e + http://repos.fedorapeople.org/repos/herlo/salt/epel\-salt.repo .ft P .fi -.SH STATES TUTORIAL, PART 1 +.RE +.SS Installation .sp -The purpose of this tutorial is to demonstrate how quickly you can configure a -system to be managed by Salt States. For detailed information about the state -system please refer to the full \fBstates reference\fP. +Salt can be installed using \fByum\fP and is available in the standard Fedora +repositories. +.SS Stable Release .sp -This tutorial will walk you through using Salt to configure a minion to run the -Apache HTTP server and to ensure the server is running. +Salt is packaged separately for the minion and the master. You\(aqll only need to +install the appropriate package for the role you need the machine to play. This +means you\(aqre going to want one master and a whole bunch of minions! .sp -\fBBefore continuing\fP make sure you have a working Salt installation by -following the \fI\%installation\fP and the \fBconfiguration\fP instructions. -.IP "Stuck?" +.nf +.ft C +yum install salt\-master +yum install salt\-minion +.ft P +.fi +.SS Configuration .sp -If you get stuck at any point, there are many ways to \fBget help from -the Salt community\fP including our mailing list and our -IRC channel. -.RE -.SS Setting up the Salt State Tree +In the sections below I\(aqll outline configuration options for both the Salt +Master and Salt Minions. +.SS Master Configuration .sp -States are stored in text files on the master and transfered to the minions on -demand via the master\(aqs File Server. The collection of state files make up the -\fIState Tree\fP. +This section outlines configuration of a Salt Master, which is used to control +other machines known as "minions" (see "Minion Configuration" for instructions +on configuring a minion). This will outline IP configuration, and a few key +configuration paths. .sp -To start using a central state system in Salt you must first set up the Salt -File Server. Edit your master config file (\fBfile_roots\fP) and -uncomment the following lines: +\fBInterface\fP +.sp +By default the Salt master listens on ports 4505 and 4506 on all interfaces +(0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the +"interface" directive as seen here: .sp .nf .ft C -file_roots: - base: - \- /srv/salt +\- #interface: 0.0.0.0 ++ interface: 10.0.0.1 .ft P .fi .sp -Restart the Salt master in order to pick up this change: +\fBEnable the Master\fP +.sp +You\(aqll also likely want to activate the Salt Master in systemd, configuring the +Salt Master to start automatically at boot. .sp .nf .ft C -% pkill salt\-master -% salt\-master \-d +systemctl enable salt\-master.service .ft P .fi -.SS Preparing the Top File .sp -On the master in the directory you specified in the previous step, create a new -file called \fBtop.sls\fP and add the following: +\fBStart the Master\fP +.sp +Once you\(aqve completed all of these steps you\(aqre ready to start your Salt +Master. You should be able to start your Salt Master now using the command +seen here: .sp .nf .ft C -base: - \(aq*\(aq: - \- webserver +systemctl start salt\-master.service .ft P .fi .sp -The \fItop file\fP is separated into environments (discussed later). The -default environment is \fBbase\fP. Under the \fBbase\fP environment a collection of -minion matches is defined; for now simply specify all hosts (\fB*\fP). -.IP "Targeting minions" +If your Salt Master doesn\(aqt start successfully, go back through each step and +see if anything was missed. Salt doesn\(aqt take much configuration (part of its +beauty!), and errors are usually simple mistakes. +.SS Minion Configuration .sp -The expressions can use any of the targeting mechanisms used by Salt — -minions can be matched by glob, pcre regular expression, or by \fBgrains\fP. For example: +Configuring a Salt Minion is surprisingly simple. Unless you have a real need +for customizing your minion configuration (which there are plenty of options if +you are so inclined!), there is one simple directive that needs to be updated. +That option is the location of the master. +.sp +By default a Salt Minion will try to connect to the dns name "salt". If you +have the ability to update DNS records for your domain you might create an A or +CNAME record for "salt" that points to your Salt Master. If you are able to do +this you likely can do without any minion configuration at all. +.sp +If you are not able to update DNS, you\(aqll simply need to update one entry in +the configuration file. Using your favorite editor, open the minion +configuration file and update the "master" entry as seen here: .sp .nf .ft C -base: - \(aqos:Fedora\(aq: - \- match: grain - \- webserver +\- #master: salt ++ master: 10.0.0.1 .ft P .fi -.RE -.SS Create an \fBsls\fP module .sp -In the same directory as your \fItop file\fP, create an empty file, called an -\fIsls module\fP, named \fBwebserver.sls\fP. Type the following and save the -file: +Simply update the master directive to the IP or hostname of your Salt Master. +Save your changes and you\(aqre ready to start your Salt Minion. Advanced +configuration options are covered in another chapter. +.sp +\fBEnable the Minion\fP +.sp +You\(aqll need to configure the minion to auto\-start at boot. You can toggle +that option through systemd. .sp .nf .ft C -apache: # ID declaration - pkg: # state declaration - \- installed # function declaration +systemctl enable salt\-minion.service .ft P .fi .sp -The first line, called the \fIID declaration\fP, is an arbitrary identifier. -In this case it defines the name of the package to be installed. \fBNOTE:\fP the -package name for the Apache httpd web server may differ on your OS or distro — -for example, on Fedora it is \fBhttpd\fP but on Debian/Ubuntu it is \fBapache2\fP. +\fBStart the Minion\fP .sp -Additionally, an ID declaration should not contain a dot, as this will produce -unpredictable output in the summary returned from a call to -\fBstate.highstate\fP. +Once you\(aqve completed all of these steps you\(aqre ready to start your Salt +Minion. You should be able to start your Salt Minion now using the command +here: .sp -The second line, called the \fIstate declaration\fP, defines which of the -Salt States we are using. In this example, we are using the \fBpkg state\fP to ensure that a given package is installed. +.nf +.ft C +systemctl start salt\-minion.service +.ft P +.fi .sp -The third line, called the \fIfunction declaration\fP defines which function -in the \fBpkg state\fP module to call. -.IP "Renderers" +If your Salt Minion doesn\(aqt start successfully, go back through each step and +see if anything was missed. Salt doesn\(aqt take much configuration (part of its +beauty!), and errors are usually simple mistakes. +.SS Tying It All Together .sp -States \fIsls\fP files can be written in many formats. Salt requires only -a simple data structure and is not concerned with how that data structure -is built. Templating languages and \fI\%DSLs\fP are a dime\-a\-dozen and everyone -has a favorite. +If you\(aqve successfully completed each of the steps above you should have a +running Salt Master and a running Salt Minion. The Minion should be configured +to point to the Master. To verify that there is communication flowing between +the Minion and Master we\(aqll run a few initial \fBsalt\fP commands. These commands +will validate the Minions RSA encryption key, and then send a test command to +the Minion to ensure that commands and responses are flowing as expected. .sp -Building the expected data structure is the job of Salt \fBrenderers\fP and they are dead\-simple to write. +\fBKey Management\fP .sp -In this tutorial we will be using YAML in Jinja2 templates which is the -default format. You can change the default by changing -\fBrenderer\fP in the master configuration file. -.RE -.SS Install the package +Salt uses AES encryption for all communication between the Master and the +Minion. This ensures that the commands you send to your Minions (your cloud) +can not be tampered with, and that communication between Master and Minion is +only done through trusted, accepted keys. .sp -Next, let\(aqs run the state we created. Open a terminal on the master and run: +Before you\(aqll be able to do any remote execution or configuration management +you\(aqll need to accept any pending keys on the Master. Run the \fBsalt\-key\fP +command to list the keys known to the Salt Master: .sp .nf .ft C -% salt \(aq*\(aq state.highstate +[root@master ~]# salt\-key \-L +Unaccepted Keys: +alpha +bravo +charlie +delta +Accepted Keys: .ft P .fi .sp -Our master is instructing all targeted minions to run \fBstate.highstate\fP. When a minion executes a highstate call it -will download the \fItop file\fP and attempt to match the expressions. When -it does match an expression the modules listed for it will be downloaded, -compiled, and executed. -.sp -Once completed, the minion will report back with a summary of all actions taken -and all changes made. -.IP "Troubleshooting Salt" -.sp -In case you don\(aqt see the expected output, the following tips can help you -narrow down the problem. -.INDENT 0.0 -.TP -.B Turn up logging -Salt can be quite chatty when you change the logging setting to -\fBdebug\fP: +This example shows that the Salt Master is aware of four Minions, but none of +the keys have been accepted. To accept the keys and allow the Minions to be +controlled by the Master, again use the \fBsalt\-key\fP command: .sp .nf .ft C -salt\-minion \-l debug +[root@master ~]# salt\-key \-A +[root@master ~]# salt\-key \-L +Unaccepted Keys: +Accepted Keys: +alpha +bravo +charlie +delta .ft P .fi -.TP -.B Run the minion in the foreground -By not starting the minion in daemon mode (\fI\-d\fP) you can view any output from the minion as it works: +.sp +The \fBsalt\-key\fP command allows for signing keys individually or in bulk. The +example above, using \fB\-A\fP bulk\-accepts all pending keys. To accept keys +individually use the lowercase of the same option, \fB\-a keyname\fP. +.SS Sending Commands +.sp +Everything should be set for you to begin remote management of your Minions. +Whether you have a few or a few\-dozen, Salt can help you manage them easily! +.sp +For final verification, send a test function from your Salt Master to your +minions. If all of your minions are properly communicating with your Master, +you should "True" responses from each of them. See the example below to send +the \fBtest.ping\fP remote command: .sp .nf .ft C -salt\-minion & +[root@master ~]# salt \(aq*\(aq test.ping +{\(aqalpha\(aq: True} .ft P .fi -.UNINDENT +.SS Where Do I Go From Here .sp -Increase the default timeout value when running \fBsalt\fP. For -example, to change the default timeout to 60 seconds: +Congratulations! You\(aqve successfully configured your first Salt Minions and are +able to send remote commands. I\(aqm sure you\(aqre eager to learn more about what +Salt can do. Depending on the primary way you want to manage your machines you +may either want to visit the section regarding Salt States, or the section on +Modules. +.SS FreeBSD +.sp +Salt was added to the FreeBSD ports tree Dec 26th, 2011 by Christer Edwards +<\fI\%christer.edwards@gmail.com\fP>. It has been tested on FreeBSD 7.4, 8.2 and 9.0 +releases. +.sp +Salt is dependent on the following additional ports. These will be installed as +dependencies of the \fBsysutils/salt\fP port. .sp .nf .ft C -salt \-t 60 +/devel/py\-yaml +/devel/py\-pyzmq +/devel/py\-Jinja2 +/devel/py\-msgpack +/security/py\-pycrypto +/security/py\-m2crypto .ft P .fi +.SS Installation .sp -For best results, combine all three: +To install Salt from the FreeBSD ports tree, use the command: .sp .nf .ft C -salt\-minion \-l debug & # On the minion -salt \(aq*\(aq state.highstate \-t 60 # On the master +cd /usr/ports/sysutils/salt && make install clean .ft P .fi -.RE -.SS Next steps -.sp -This tutorial focused on getting a simple Salt States configuration working. -\fBPart 2\fP will build on this example to cover more advanced -\fIsls\fP syntax and will explore more of the states that ship with Salt. -.SH STATES TUTORIAL, PART 2 .sp -This tutorial builds on the topic covered in \fBpart 1\fP. It is -recommended that you begin there. +Once the port is installed you\(aqll need to make a few configuration changes. +These include defining the IP to bind to (optional), and some configuration +path changes to make salt fit more natively into the FreeBSD filesystem tree. +.SS Configuration .sp -In the last Salt States tutorial we covered the basics of installing a package. -In this tutorial we will modify our \fBwebserver.sls\fP file to be more -complicated, have requirements, and use even more Salt States. -.SS Call multiple States +In the sections below I\(aqll outline configuration options for both the Salt +Master and Salt Minions. .sp -You can specify multiple \fIstate declarations\fP under -an \fIID declaration\fP. For example, a quick modification to our -\fBwebserver.sls\fP to also start Apache if it is not running: +The Salt port installs two sample configuration files, \fBsalt/master.sample\fP +and \fBsalt/minion.sample\fP (these should be installed in \fB/usr/local/etc/\fP, +unless you use a different \fB%%PREFIX%%\fP). You\(aqll need to copy these +.sample files into place and make a few edits. First, copy them into place +as seen here: .sp .nf .ft C -apache: - pkg: - \- installed - service: - \- running +cp /usr/local/etc/salt/master.sample /usr/local/etc/salt/master +cp /usr/local/etc/salt/minion.sample /usr/local/etc/salt/minion .ft P .fi .sp -Try stopping Apache before running \fBstate.highstate\fP once again and observe -the output. -.SS Expand the SLS module +Note: You\(aqll only need to copy the config for the service you\(aqre going to run. .sp -As you have seen, sls modules are appended with the file extension \fB.sls\fP and -are referenced by name starting at the root of the state tree. An SLS module -can be also defined as a directory. Demonstrate that now by creating a -directory named \fBwebserver\fP and moving and renaming \fBwebserver.sls\fP to -\fBwebserver/init.sls\fP. Your state directory should now resemble: +Once you\(aqve copied the config into place you\(aqll need to make changes specific +to your setup. Below I\(aqll outline suggested configuration changes to the +Master, after which I\(aqll outline configuring the Minion. +.SS Master Configuration +.sp +This section outlines configuration of a Salt Master, which is used to control +other machines known as "minions" (see "Minion Configuration" for instructions +on configuring a minion). This will outline IP configuration, and a few key +configuration paths. +.sp +\fBInterface\fP +.sp +By default the Salt master listens on ports 4505 and 4506 on all interfaces +(0.0.0.0). If you have a need to bind Salt to a specific IP, redefine the +"interface" directive as seen here. .sp .nf .ft C -|\- top.sls -\(ga\- webserver/ - \(ga\- init.sls +\- #interface: 0.0.0.0 ++ interface: 10.0.0.1 .ft P .fi -.IP "Organizing SLS modules" .sp -You can place additional \fB.sls\fP files in a state file directory. This -affords much cleaner organization of your state tree on the filesystem. For -example, if we created a \fBwebserver/django.sls\fP file that module would be -referenced as \fBwebserver.django\fP. -.sp -In addition, States provide powerful includes and extending functionality -which we will cover in \fBPart 3\fP. -.RE -.SS Require other states +\fBrc.conf\fP .sp -We now have a working installation of Apache so let\(aqs add an HTML file to -customize our website. It isn\(aqt exactly useful to have a website without a -webserver so we don\(aqt want Salt to install our HTML file until Apache is -installed and running. Include the following at the bottom of your -\fBwebserver/init.sls\fP file: +Last but not least you\(aqll need to activate the Salt Master in your rc.conf +file. Using your favorite editor, open \fB/etc/rc.conf\fP or +\fB/etc/rc.conf.local\fP and add this line. .sp .nf .ft C -apache: - pkg: - \- installed - service: - \- running - -/var/www/index.html: # ID declaration - file: # state declaration - \- managed # function - \- source: salt://webserver/index.html # function arg - \- require: # requisite declaration - \- pkg: apache # requisite reference ++ salt_master_enable="YES" .ft P .fi .sp -Again in \fBline 1\fP is the \fIID declaration\fP. In this example it is the -location we want to install our custom HTML file. (\fBNote:\fP the default -location that Apache serves may differ from the above on your OS or distro. -\fB/srv/www\fP could also be a likely place to look.) +\fBStart the Master\fP .sp -\fBLine 2\fP the \fIstate declaration\fP. This example uses the Salt \fBfile -state\fP. +Once you\(aqve completed all of these steps you\(aqre ready to start your Salt +Master. The Salt port installs an rc script which should be used to manage your +Salt Master. You should be able to start your Salt Master now using the command +seen here: .sp -\fBLine 3\fP is the \fIfunction declaration\fP. The \fBmanaged function\fP will download a file from the master and install it -in the location specified. +.nf +.ft C +service salt_master start +.ft P +.fi .sp -\fBLine 4\fP is a \fIfunction arg declaration\fP which, in this example, passes -the \fBsource\fP argument to the \fBmanaged function\fP. +If your Salt Master doesn\(aqt start successfully, go back through each step and +see if anything was missed. Salt doesn\(aqt take much configuration (part of its +beauty!), and errors are usually simple mistakes. +.SS Minion Configuration .sp -\fBLine 5\fP is a \fIrequisite declaration\fP. +Configuring a Salt Minion is surprisingly simple. Unless you have a real need +for customizing your minion configuration (which there are plenty of options if +you are so inclined!), there is one simple directive that needs to be updated. +That option is the location of the master. .sp -\fBLine 6\fP is a \fIrequisite reference\fP which refers to a state and an ID. -In this example, it is referring to the \fBID declaration\fP from our example in -\fBpart 1\fP. This declaration tells Salt not to install the HTML -file until Apache is installed. +By default a Salt Minion will try to connect to the dns name "salt". If you +have the ability to update DNS records for your domain you might create an A or +CNAME record for "salt" that points to your Salt Master. If you are able to do +this you likely can do without any minion configuration at all. .sp -Next, create the \fBindex.html\fP file and save it in the \fBwebserver\fP -directory: +If you are not able to update DNS, you\(aqll simply need to update one entry in +the configuration file. Using your favorite editor, open the minion +configuration file and update the "master" entry as seen here. .sp .nf .ft C -<html> - <head><title>Salt rocks</title></head> - <body> - <h1>This file brought to you by Salt</h1> - </body> -</html> +\- #master: salt ++ master: 10.0.0.1 .ft P .fi .sp -Last, call \fBstate.highstate\fP again and the -minion will fetch and execute the highstate as well as our HTML file from the -master using Salt\(aqs File Server: +Simply update the master directive to the IP or hostname of your Salt Master. +Save your changes and you\(aqre ready to start your Salt Minion. Advanced +configuration options are covered in another chapter. +.sp +\fBrc.conf\fP +.sp +Before you\(aqre able to start the Salt Minion you\(aqll need to update your rc.conf +file. Using your favorite editor open \fB/etc/rc.conf\fP or +\fB/etc/rc.conf.local\fP and add this line. .sp .nf .ft C -salt \(aq*\(aq state.highstate ++ salt_minion_enable="YES" .ft P .fi .sp -Verify that Apache is now serving your custom HTML. -.IP "\fBrequire\fP vs. \fBwatch\fP" -.sp -There are two \fIrequisite declarations\fP, -“require” and “watch”. Not every state supports “watch”. The \fBservice -state\fP does support “watch” and will restart a -service based on the watch condition. -.sp -For example, if you use Salt to install an Apache virtual host -configuration file and want to restart Apache whenever that file is changed -you could modify our Apache example from earlier as follows: +Once you\(aqve completed all of these steps you\(aqre ready to start your Salt +Minion. The Salt port installs an rc script which should be used to manage your +Salt Minion. You should be able to start your Salt Minion now using the command +seen here. .sp .nf .ft C -/etc/httpd/extra/httpd\-vhosts.conf: - file: - \- managed - \- source: salt://webserver/httpd\-vhosts.conf - -apache: - pkg: - \- installed - service: - \- running - \- watch: - \- file: /etc/httpd/extra/httpd\-vhosts.conf +service salt_minion start .ft P .fi .sp -If the pkg and service names differ on your OS or distro of choice you can -specify each one separately using a \fIname declaration\fP which -explained in \fBPart 3\fP. -.RE -.SS Next steps -.sp -In \fBpart 3\fP we will discuss how to use includes, extends and -templating to make hugely complicated State Tree configurations dead\-simple. -.SH STATES TUTORIAL, PART 3 +If your Salt Minion doesn\(aqt start successfully, go back through each step and +see if anything was missed. Salt doesn\(aqt take much configuration (part of its +beauty!), and errors are usually simple mistakes. +.SS Tying It All Together .sp -This tutorial builds on the topic covered in \fBpart 2\fP. It is -recommended that you begin there. +If you\(aqve successfully completed each of the steps above you should have a +running Salt Master and a running Salt Minion. The Minion should be configured +to point to the Master. To verify that there is communication flowing between +the Minion and Master we\(aqll run a few initial \fBsalt\fP commands. These commands +will validate the Minions RSA encryption key, and then send a test command to +the Minion to ensure that commands and responses are flowing as expected. .sp -This tutorial will cover more advanced templating and configuration techniques -for \fBsls\fP files. -.SS Templating SLS modules +\fBKey Management\fP .sp -SLS modules may require programming logic or inline executions. This is -accomplished with module templating. The default module templating system used -is \fI\%Jinja2\fP and may be configured by changing the \fBrenderer\fP -value in the master config. +Salt uses AES encryption for all communication between the Master and the +Minion. This ensures that the commands you send to your Minions (your cloud) +can not be tampered with, and that communication between Master and Minion is +only done through trusted, accepted keys. .sp -All states are passed through a templating system when they are initially read, -so all that is required to make use of the templating system is to add some -templating code. An example of an sls module with templating may look like -this: +Before you\(aqll be able to do any remote execution or state management you\(aqll +need to accept any pending keys on the Master. Run the \fBsalt\-key\fP command to +list the keys known to the Salt Master: .sp .nf .ft C -{% for usr in \(aqmoe\(aq,\(aqlarry\(aq,\(aqcurly\(aq %} -{{ usr }}: - user: - \- present -{% endfor %} +[root@master ~]# salt\-key \-L +Unaccepted Keys: +alpha +bravo +charlie +delta +Accepted Keys: .ft P .fi .sp -This templated sls file once generated will look like this: +This example shows that the Salt Master is aware of four Minions, but none of +the keys have been accepted. To accept the keys and allow the Minions to be +controlled by the Master, again use the \fBsalt\-key\fP command: .sp .nf .ft C -moe: - user: - \- present -larry: - user: - \- present -currly: - user: - \- present +[root@master ~]# salt\-key \-A +[root@master ~]# salt\-key \-L +Unaccepted Keys: +Accepted Keys: +alpha +bravo +charlie +delta .ft P .fi -.SS Using Grains in SLS modules .sp -Often times a state will need to behave differently on different systems. -\fBSalt grains\fP can be used from within sls modules. An object -called \fBgrains\fP is made available in the template context: +The \fBsalt\-key\fP command allows for signing keys individually or in bulk. The +example above, using \fB\-A\fP bulk\-accepts all pending keys. To accept keys +individually use the lowercase of the same option, \fB\-a keyname\fP. +.SS Sending Commands +.sp +Everything should be set for you to begin remote management of your Minions. +Whether you have a few or a few\-dozen, Salt can help you manage them easily! +.sp +For final verification, send a test function from your Salt Master to your +minions. If all of your minions are properly communicating with your Master, +you should "True" responses from each of them. See the example below to send +the \fBtest.ping\fP remote command. .sp .nf .ft C -apache: - pkg: - {% if grains[\(aqos\(aq] == \(aqRedHat\(aq %} - \- name: httpd - {% elif grains[\(aqos\(aq] == \(aqUbuntu\(aq %} - \- name: apache2 - {% endif %} - \- installed +[root@master ~]# salt \(aqalpha\(aq test.ping +{\(aqalpha\(aq: True} .ft P .fi -.SS Calling Salt modules from templates +.SS Where Do I Go From Here .sp -All of the Salt modules loaded by the minion ave available within the -templating system. This allows data to be gathered in real time, on the target -system. It also allows for shell commands to be run easily from within the sls -modules. +Congratulations! You\(aqve successfully configured your first Salt Minions and are +able to send remote commands. I\(aqm sure you\(aqre eager to learn more about what +Salt can do. Depending on the primary way you want to manage your machines you +may either want to visit the section regarding Salt States, or the section on +Modules. +.SS Gentoo .sp -The Salt module functions are also made available in the template context as -\fBsalt\fP: +Salt can be easily installed on Gentoo: .sp .nf .ft C -{% for usr in \(aqmoe\(aq,\(aqlarry\(aq,\(aqcurly\(aq %} -{{ usr }}: - group: - \- present - user: - \- present - \- gid: {{ salt[\(aqfile.group_to_gid\(aq](usr) }} - \- require: - \- group: {{ usr }} -{% endfor %} +emerge pyyaml m2crypto pycrypto jinja pyzmq .ft P .fi .sp -Below is another example that calls an arbitrary command in order to grab the -mac addr for eth0: +Then download and install from source: +.INDENT 0.0 +.IP 1. 3 +Download the latest source tarball from the \fI\%GitHub downloads\fP directory for +the Salt project. +.IP 2. 3 +Untar the tarball and run the \fBsetup.py\fP as root: +.UNINDENT .sp .nf .ft C -salt[\(aqcmd.run\(aq](\(aqifconfig eth0 | grep HWaddr | cut \-d" " \-f10\(aq) +tar xf salt\-<version>.tar.gz +cd salt\-<version> +python setup.py install .ft P .fi -.SS Advanced SLS module syntax +.SH CONFIGURING SALT .sp -Last we will cover some incredibly useful techniques for more complex State -trees. -.SS \fIInclude declaration\fP +Salt configuration is very simple. The default configuration for the +\fImaster\fP will work for most installations and the only requirement for +setting up a \fIminion\fP is to set the location of the master in the minion +configuration file. +.INDENT 0.0 +.TP +.B master +The Salt master is the central server that all minions connect to. You +run commands on the minions through the master and minions send data +back to the master (unless otherwise redirected with a \fBreturner\fP). It is started with the +\fBsalt\-master\fP program. +.TP +.B minion +Salt minions are the potentially hundreds or thousands of servers that +you query and control from the master. +.UNINDENT .sp -You have seen an example of how to spread a Salt tree across several files but -in order to be able to have \fIrequisite references\fP -span multiple files you must use a \fIinclude declaration\fP. For example: +The configuration files will be installed to \fB/etc/salt\fP and are named +after the respective components, \fB/etc/salt/master\fP and +\fB/etc/salt/minion\fP. .sp -\fBpython\-libs.sls\fP: +To make a minion check into the correct master simply edit the +\fBmaster\fP variable in the minion configuration file to reference +the master DNS name or IPv4 address. +.SS Running Salt +.INDENT 0.0 +.IP 1. 3 +Start the master in the foreground (to daemonize the process, pass the +\fI\-d flag\fP): .sp .nf .ft C -python\-dateutil: - pkg: - \- installed +# salt\-master .ft P .fi -.sp -\fBdjango.sls\fP: +.IP 2. 3 +Start the minion in the foreground (to daemonize the process, pass the +\fI\-d flag\fP): .sp .nf .ft C -include: - \- python\-libs - -django: - pkg: - \- installed - \- require: - \- pkg: python\-dateutil +# salt\-minion .ft P .fi -.SS \fIExtend declaration\fP -.sp -You can modify previous declarations by using a \fIextend declaration\fP. For -example the following modifies the Apache tree to also restart Apache when the -vhosts file is changed: +.UNINDENT +.IP "Having trouble?" .sp -\fBapache.sls\fP: +The simplest way to troubleshoot Salt is to run the master and minion in +the foreground with \fIlog level\fP set to \fBdebug\fP: .sp .nf .ft C -apache: - pkg: - \- installed +salt\-master \-\-log\-level=debug .ft P .fi +.RE +.IP "Run as an unprivileged (non\-root) user?" .sp -\fBmywebsite.sls\fP: +To run Salt as another user, specify \fB\-\-user\fP in the command +line or assign \fBuser\fP in the +\fBconfiguration file\fP. +.RE +.sp +There is also a full \fBtroubleshooting guide\fP +available. +.SS Manage Salt public keys +.sp +Salt manages authentication with RSA public keys. The keys are managed on the +\fImaster\fP via the \fBsalt\-key\fP command. Once a \fIminion\fP +checks into the master the master will save a copy of the minion key. Before +the master can send commands to the minion the key needs to be "accepted". +.INDENT 0.0 +.IP 1. 3 +List the accepted and unaccepted salt keys: .sp .nf .ft C -include: - \- apache - -extend: - apache: - service: - \- watch: - \- file: /etc/httpd/extra/httpd\-vhosts.conf - -/etc/httpd/extra/httpd\-vhosts.conf: - file: - \- managed - \- source: salt://httpd\-vhosts.conf +salt\-key \-L .ft P .fi -.SS \fIName declaration\fP -.sp -You can override the \fIID declaration\fP by using a \fIname -declaration\fP. For example the previous example is a bit more maintainable if -rewritten as the following: -.sp -\fBmywebsite.sls\fP: +.IP 2. 3 +Accept a minion key: .sp .nf .ft C -include: - \- apache - -extend: - apache - service: - \- watch: - \- file: mywebsite - -mywebsite: - file: - \- managed - \- name: /etc/httpd/extra/httpd\-vhosts.conf - \- source: salt://httpd\-vhosts.conf +salt\-key \-a <minion id> .ft P .fi -.SS \fINames declaration\fP .sp -Even more powerful is using a \fInames declaration\fP to override the -\fIID declaration\fP for multiple states at once. This often can remove the -need for looping in a template. For example, the first example in this tutorial -can be rewritten without the loop: +or accept all unaccepted minion keys: .sp .nf .ft C -stooges: - user: - \- present - \- names: - \- moe - \- larry - \- curly +salt\-key \-A .ft P .fi -.SS Continue learning -.sp -The best way to continue learing about Salt States is to read through the -\fBreference documentation\fP and to look through examples -of existing \fIstate trees\fP. You can find examples in the -\fI\%salt-states repository\fP and please send a pull\-request on GitHub with any -state trees that you build and want to share! +.UNINDENT +.IP "See also" .sp -If you have any questions, suggestions, or just want to chat with other people -who are using Salt we have an \fBactive community\fP. -.SH COMMUNITY +\fBsalt\-key manpage\fP +.RE +.SH TARGETING +.INDENT 0.0 +.TP +.B Targeting +Specifying which minions should run a command or execute a state by +matching against hostnames, or system information, or defined groups, +or even combinations thereof. +.UNINDENT .sp -Join the Salt! +For example the command \fBsalt web1 apache.signal restart\fP to restart the +Apache httpd server specifies the machine \fBweb1\fP as the target and the +command will only be run on that one minion. .sp -There are many ways to participate in and communicate with the Salt community. +Similarly when using states, the following \fItop file\fP specifies that only +the \fBweb1\fP minion should execute the contents of \fBwebserver.sls\fP: .sp -Salt has an active IRC channel and a mailing list. -.SS Mailing List +.nf +.ft C +base: + \(aqweb1\(aq: + \- webserver +.ft P +.fi .sp -Join the \fI\%salt-users mailing list\fP. It is the best place to ask questions -about Salt and see whats going on with Salt development! The Salt mailing list -is hosted by Google Groups. It is open to new members. +There are many ways to target individual minions or groups of minions in Salt: +.SS Matching the \fBminion id\fP +.INDENT 0.0 +.TP +.B minion id +A unique identifier for a given minion. By default the minion id is the +FQDN of that host but this can be overridden. +.UNINDENT .sp -\fI\%http://groups.google.com/group/salt-users\fP -.SS IRC +Each minion needs a unique identifier. By default when a minion starts for the +first time it chooses its FQDN as that +identifier. The minion id can be overridden via the minion\(aqs \fBid\fP +configuration setting. +.IP Tip +minion id and minion keys .sp -The \fB#salt\fP IRC channel is hosted on the popular \fI\%Freenode\fP network. You -can use the \fI\%Freenode webchat client\fP right from your browser. -.SS Follow on Github +The \fIminion id\fP is used to generate the minion\(aqs public/private keys +and if it ever changes the master must then accept the new key as though +the minion was a new host. +.RE +.SS Globbing .sp -The Salt code is developed via Github. Follow Salt for constant updates on what -is happening in Salt development: +The default matching that Salt utilizes is shell\-style globbing around the +\fIminion id\fP. This also works for states in the \fItop file\fP. +.IP Note +You must wrap \fBsalt\fP calls that use globbing in single\-quotes to +prevent the shell from expanding the globs before Salt is invoked. +.RE .sp -\fI\%https://github.com/saltstack/salt\fP -.SS The Red45 Blog +Match all minions: .sp -News and thoughts on Salt and related projects is often posted on Thomas\(aq blog -\fI\%The Red45\fP: +.nf +.ft C +salt \(aq*\(aq test.ping +.ft P +.fi .sp -\fI\%http://red45.wordpress.com/\fP -.SS Follow on ohloh +Match all minions in the example.net domain or any of the example domains: .sp -\fI\%https://www.ohloh.net/p/salt\fP -.SS Developing Salt +.nf +.ft C +salt \(aq*.example.net\(aq test.ping +salt \(aq*.example.*\(aq test.ping +.ft P +.fi .sp -If you want to help develop Salt there is a great need and your patches are -welcome! +Match all the \fBwebN\fP minions in the example.net domain +(\fBweb1.example.net\fP, \fBweb2.example.net\fP … \fBwebN.example.net\fP): .sp -To assist in Salt development, you can help in a number of ways. -.SS Posting patches to the mailing list +.nf +.ft C +salt \(aqweb?.example.net test.ping +.ft P +.fi .sp -If you have a patch for Salt, please format it via \fBgit format\-patch\fP and -send it to the Salt users mailing list. This allows the patch to give you the -contributor the credit for your patch, and gives the Salt community an archive -of the patch and a place for discussion. -.SS Setting a Github pull request +Match the \fBweb1\fP through \fBweb5\fP minions: .sp -This is probably the preferred method for contributions, simply create a Github -fork, commit your changes to the fork, and then open up a pull request. -.SS Contributions Welcome! +.nf +.ft C +salt \(aqweb[1\-5]\(aq test.ping +.ft P +.fi .sp -The goal here it to make contributions clear, make sure there is a trail for -where the code has come from, but most importantly, to give credit where credit -is due! -.SH INTRODUCTION TO EXTENDING SALT +Match the \fBweb\-x\fP, \fBweb\-y\fP, and \fBweb\-z\fP minions: .sp -Salt is made to be used, and made to be extended. The primary goal of Salt is -to provide a foundation which can be used to solve problems. And the goal of -Salt is to not assume what those problems might be. +.nf +.ft C +salt \(aqweb\-[x\-z]\(aq test.ping +.ft P +.fi +.SS Regex .sp -One of the greatest benefit of developing Salt has been the vast array of ways -in which people have wanted to use it, while the original intention was as a -communication layer for a cloud controller Salt has been extended to facilitate -so much more. -.SS Client API +Minions can be matched using Perl\-compatible regular expressions (which is +globbing on steroids and a ton of caffeine). .sp -The primary interface used to extend salt, is to simply use it. Salt executions -can be called via the Salt client api, making programming master side solutions -with Salt is easy. -.SS Adding Loadable Plugins +Match both \fBweb1\-prod\fP and \fBweb1\-devel\fP minions: .sp -Salt is comprised of a core platform that loads many types of easy to write -plugins. The idea is to enable all of the breaking points in the salt processes -to have a point of pluggable interaction. This means that all of the main -features of Salt can be extended, modified or used. +.nf +.ft C +salt \-E \(aqweb1\-(prod|devel)\(aq test.ping +.ft P +.fi .sp -The breaking points and helping interfaces span from convenience master side -executions to manipulating the flow of how data is handled by Salt. -.SS Minion Execution Modules +When using regex in a states \fItop file\fP you must specify the matcher as +the first option. The following example executes the contents of +\fBwebserver.sls\fP on the above\-mentioned minions. .sp -The minion execution modules or just \fBmodules\fP are the core to what salt is -and does. These modules are found in: +.nf +.ft C +base: + \(aqweb1\-(prod|devel)\(aq: + \- match: pcre + \- webserver +.ft P +.fi +.SS Lists .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/modules\fP +At the most basic you can specify a flat list of minion ids: .sp -These modules are what is called by the salt command line and the salt client -api. Adding modules is done by simply adding additional python modules to the -modules directory and restarting the minion. +.nf +.ft C +salt \-L \(aqweb1,web2,web3\(aq test.ping +.ft P +.fi .SS Grains .sp -Salt grains, or "grains of truth" are bits of static information that are -generated when the minion starts. This information is useful when determining -what package manager to default to, or where certain configuration files are -stored on the minion. +Salt comes with an interface to derive information about the underlying system. +This is called the grains interface, because it presents salt with grains of +information. .sp -The Salt grains are the interface used for auto detection and dynamic assignment -of execution modules and types to specific salt minions. +The grains interface is made available to Salt modules and components so that +the right salt minion commands are automatically available on the right +systems. .sp -The code used to generate the Salt grains can be found here: +It is important to remember that grains are bits of information loaded when +the salt minion starts, so this information is static. This means that the +information in grains is unchanging, therefore the nature of the data is +static. So grains information are things like the running kernel, or the +operating system. +.SS Grains in the Minion Config .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/grains\fP -.SS States +Grains can also be statically assigned within the minion configuration file. +Just add the option \fBgrains\fP and pass options to it: .sp -Salt supports state enforcement, this makes Salt a high speed and very efficient -solution for system configuration management. +.nf +.ft C +grains: + roles: + \- webserver + \- memcache + deployment: datacenter4 + cabinet: 13 + cab_u: 14\-15 +.ft P +.fi .sp -States can be easily added to Salt by dropping a new state module in: +Then status data specific to your servers can be retrieved via Salt, or used +inside of the state system for matching. It also makes targeting, in the case +of the example above, simply based on specific data about your deployment. +.SS Writing Grains .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/states\fP -.SS Renderers +Grains are easy to write. The grains interface is derived by executing +all of the "public" functions found in the modules located in the grains +package or the custom grains directory. The functions in the modules of +the grains must return a python dict, where the keys in the dict are the +names of the grains and the values are the values. .sp -Salt states are controlled by simple data structures, these structures can be -abstracted in a number of ways. While the default is to be in a yaml file -wrapped in a jinja template, any abstraction can be used. This means that any -format that can be dreamed is possible, so long as a renderer is written for -it. +Custom grains should be placed in a \fB_grains\fP directory located under +your \fBfile_roots\fP. Before adding a grain to salt, consider +what the grain is and remember that grains need to be static data. +.SS Examples of Grains .sp -The existing renderers can be found here: +The core module in the grains package is where the main grains are loaded by +the salt minion and the principal example of how to write grains: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/renderers\fP -.SS Returners +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/grains/core.py\fP +.SS Syncing Grains .sp -The salt commands all produce a return value, that return value is sent to the -salt master by default, but it can be sent anywhere. The returner interface -makes it programmatically possible for the information to be sent to anything -from an SQL or NOSQL database, to a custom application made to use Salt. +Syncing grains can be done a number of ways, they are automatically synced when +state.highstate is called, or the grains can be synced and reloaded by calling +the saltutil.sync_grains or saltutil.sync_all functions. +.SS Node groups +.INDENT 0.0 +.TP +.B Node group +A predefined group of minions declared in the master configuration file +\fBnodegroups\fP setting as a \fIcompound target\fP. +.UNINDENT .sp -The existing returners can be found here: +For example, in the master config file \fBnodegroups\fP setting: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/returners\fP -.SS Runners +.nf +.ft C +nodegroups: + group1: \(aqL@foo.domain.com,bar.domain.com,baz.domain.com and bl*.domain.com\(aq + group2: \(aqG@os:Debian and foo.domain.com\(aq +.ft P +.fi .sp -Sometimes a certain application can be made to execute and run from the -existing salt command line. This is where the salt runners come into play. -The Salt Runners what is called by the salt\-run command and are meant to -act as a generic interface for encapsulating master side executions. +Specify a nodegroup via the \fB\-N\fP option at the command\-line: .sp -Existing Salt runners are located here: +.nf +.ft C +salt \-N group1 test.ping +.ft P +.fi .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/runners\fP -.SH MODULES +Specify a nodegroup with \fB\- match: nodegroup\fP in a \fItop file\fP: .sp -Salt modules are the functions called by the \fBsalt\fP command. -.IP "See also" +.nf +.ft C +base: + group1: + \- match: nodegroup + \- webserver +.ft P +.fi +.SS Compound matchers +.INDENT 0.0 +.TP +.B Compound matcher +A combination of many target definitions that can be combined with +boolean operators. +.UNINDENT .sp -\fIFull list of builtin modules\fP +Compound matchers allow very granular minion targeting using any of the +previously discussed matchers. The default matcher is a glob, as usual. If +something other than a glob is used preface it with the letter denoting the +type. Matchers can be joined using boolean \fBand\fP, \fBor\fP, and \fBnot\fP +operators. .sp -Salt ships with many modules that cover a wide variety of tasks. +For example, the following command matches all minions that have a hostname +that begins with "webserv" and that are running Debian or it matches any +minions that have a hostname that matches the regular expression +\fBweb\-dc1\-srv.*\fP: +.sp +.nf +.ft C +salt \-C \(aqwebserv* and G@os:Debian or E@web\-dc1\-srv.*\(aq test.ping +.ft P +.fi +.sp +That same example expressed in a \fItop file\fP looks like the following: +.sp +.nf +.ft C +base: + \(aqwebserv* and G@os:Debian or E@web\-dc1\-srv.*\(aq: + \- match: compound + \- webserver +.ft P +.fi +.SH REMOTE EXECUTION TUTORIAL +.sp +\fBBefore continuing\fP make sure you have a working Salt installation by +following the \fI\%installation\fP and the \fBconfiguration\fP instructions. +.IP "Stuck?" +.sp +If you get stuck at any point, there are many ways to \fBget help from +the Salt community\fP including our mailing list and our +IRC channel. .RE -.SS Easy Modules to write +.SS Order your minions around .sp -Salt modules are amazingly simple to write, just write a regular Python module -or a regular Cython module and place it in the \fBsalt/modules\fP directory. +Now that you have a \fImaster\fP and at least one \fIminion\fP +communicating with each other you can perform commands on the minion via the +\fBsalt\fP command. Salt calls are comprised of three main components: .sp -Since Salt modules are just Python/Cython modules there are no restraints as to -what you can put inside of a salt module, and if a Salt module has errors and -cannot import the Salt minion will continue to load without issue, the module -with errors will simply be omitted. +.nf +.ft C +salt \(aq<target>\(aq <function> [arguments] +.ft P +.fi +.IP "See also" .sp -If adding a Cython module the file must be named \fB<modulename>.pyx\fP so that -the loader knows that the module needs to be imported as a Cython module. The -compilation of the Cython module is automatic and happens when the minion -starts, so only the \fB*.pyx\fP file is required. -.SS Cross Calling Modules +\fBsalt manpage\fP +.RE +.SS target .sp -All of the salt modules are available to each other, and can be "cross called". -This means that when creating a module functions in modules which already exist -can be called. +The target component allows you to filter which minions should run the +following function. The default filter is a glob on the minion id. E.g.: .sp -The variable \fB__salt__\fP is packed into the modules after they are loaded into -the salt minion. This variable is a python dictionary of all of the salt -functions, laid out in the same way that they are made available to the salt -command. +.nf +.ft C +salt \(aq*\(aq test.ping +salt \(aq*.example.org\(aq test.ping +.ft P +.fi .sp -Salt modules can be cross called by accessing the value in the \fB__salt__\fP -dict: +Targets can be based on minion system information using the grains system: .sp .nf .ft C -def foo(bar): - return __salt__[\(aqcmd.run\(aq](bar) +salt \-G \(aqos:Ubuntu\(aq test.ping .ft P .fi +.IP "See also" .sp -This code will call the Salt cmd module\(aqs run function and pass the argument -\fBbar\fP. -.SS Preloaded Modules Data +\fBGrains system\fP +.RE .sp -When interacting with modules often it is nice to be able to read information -dynamically about the minion, or load in configuration parameters for a module. -Salt allows for different types of data to be loaded into the modules by the -minion, as of this writing Salt loads information gathered from the Salt Grains -system and from the minion configuration file. -.SS Grains Data +Targets can be filtered by regular expression: .sp -The Salt minion detects information about the system when started. This allows -for modules to be written dynamically with respect to the underlying hardware -and OS. This information is referred to as Salt Grains, or "grains of salt". -The Grains system was introduced to replace Facter, since relying on a Ruby -application from a Python application was both slow and inefficient. Grains -support replaces Facter in all releases after 0.8 +.nf +.ft C +salt \-E \(aqvirtmach[0\-9]\(aq test.ping +.ft P +.fi .sp -The values detected by the Salt Grains on the minion are available in a dict by -the name of \fB__grains__\fP and can be accessed from within callable objects in -the Python modules. +Finally, targets can be explicitly specified in a list: .sp -To see the contents of the grains dict for a given system in your deployment -run the \fBgrains.items()\fP function: +.nf +.ft C +salt \-L foo,bar,baz,quo test.ping +.ft P +.fi +.SS function +.sp +A function is some functionality provided by a module. Salt ships with a large +collection of available functions. List all available functions on your +minions: .sp .nf .ft C -salt \(aqhostname\(aq grains.items +salt \(aq*\(aq sys.doc .ft P .fi .sp -To use the \fB__grains__\fP dict simply call it as a Python dict from within your -code, an excellent example is available in the Grains module: -\fBsalt.modules.grains\fP. -.SS Module Configuration +Here are some examples: .sp -Since parameters for configuring a module may be desired, Salt allows for -configuration information stored in the main minion config file to be passed to -the modules. +Show all currently available minions: .sp -Since the minion configuration file is a yaml document, arbitrary configuration -data can be passed in the minion config that is read by the modules. It is -\fBstrongly\fP recommended that the values passed in the configuration file match -the module. This means that a value intended for the \fBtest\fP module should be -named \fBtest.<value>\fP. +.nf +.ft C +salt \(aq*\(aq test.ping +.ft P +.fi .sp -Configuration also requires that default configuration parameters need to be -loaded as well. This can be done simply by adding the \fB__opts__\fP dict to the -top level of the module. +Run an arbitrary shell command: .sp -The test module contains usage of the module configuration, and the default -configuration file for the minion contains the information and format used to -pass data to the modules. \fBsalt.modules.test\fP, \fBconf/minion\fP. -.SS Printout Configuration +.nf +.ft C +salt \(aq*\(aq cmd.run \(aquname \-a\(aq +.ft P +.fi +.IP "See also" .sp -Since module functions can return different data, and the way the data is -printed can greatly change the presentation, Salt has a printout -configuration. +\fBthe full list of modules\fP +.RE +.SS arguments .sp -When writing a module the \fB__outputter__\fP dict can be declared in the module. -The \fB__outputter__\fP dict contains a mapping of function name to Salt -Outputter. +Space\-delimited arguments to the function: .sp .nf .ft C -__outputter__ = { - \(aqrun\(aq: \(aqtxt\(aq - } +salt \(aq*\(aq cmd.exec_code python \(aqimport sys; print sys.version\(aq .ft P .fi +.SH HOW DO I USE SALT STATES? .sp -This will ensure that the text outputter is used. -.SS Virtual Modules +Simplicity, Simplicity, Simplicity .sp -Sometimes a module should be presented in a generic way. A good example of this -can be found in the package manager modules. The package manager changes from -one operating system to another, but the salt module that interfaces with the -package manager can be presented in a generic way. +Many of the most powerful and useful engineering solutions are founded on +simple principals, the Salt SLS system strives to do just that. K.I.S.S. .sp -The salt modules for package managers all contain a \fB__virtual__\fP function -which is called to define what systems the module should be loaded on. +The core of the Salt State system is the SLS, or the SaLt State file. The SLS +is a representation of the state in which a system should be in, and is set up +to contain this data simply. This is often called configuration management. +.SS It is All Just Data .sp -The \fB__virtual__\fP function is used to return either a string or False. If -False is returned then the module is not loaded, if a string is returned then -the module is loaded with the name of the string. +Before delving into the particulars, it will help to understand that the SLS +is just a data structure under the hood. While understanding that the SLS is +just a data structure is not at all critical to understand to make use Salt +States, it should help bolster the understanding of where the real power is. .sp -This means that the package manager modules can be presented as the pkg module -regardless of what the actual module is named. +SLS files are therefore, in reality, just dictionaries, lists, strings and +numbers. By using this approach Salt can be much more flexible. As someone +writes more state files, it becomes clear exactly what is being written. The +result is a system that is easy to understand, yet grows with the needs of +the admin or developer. .sp -The package manager modules are the best example of using the \fB__virtual__\fP -function: -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/modules/pacman.py\fP -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/modules/yumpkg.py\fP -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/modules/apt.py\fP -.SS Documentation +In the section titled "State Data Structures" a reference exists, explaining +in depth how the data is laid out. +.SS Default Data \- YAML .sp -Salt modules are self documenting, the \fBsys.doc()\fP function will return the -documentation for all available Facter modules: +By default Salt represents the SLS data in what is one of the simplest +serialization formats available \- YAML. +.sp +A typical SLS file will often look like this in YAML: .sp .nf .ft C -salt \(aq*\(aq sys.doc + apache: + pkg: + \- installed + service: + \- running + \- require: + \- pkg: apache .ft P .fi .sp -This function simple prints out the docstrings found in the modules, when -writing salt modules, please follow the formating conventions for docstrings as -they appear in the other modules. -.SS Adding Documentation to Salt Modules +This SLS data will ensure that the package named apache is installed, and +that the apache service is running. The components can be explained in a +simple way. .sp -Since life is much better with documentation, it is strongly suggested that -all Salt modules have documentation added. Any Salt modules submitted for -inclusion in the main distribution of Salt will be required to have -documentation. +The first like it the ID for a set of data, and it is called the ID +Declaration. This ID sets the name of the thing that needs to be manipulated. .sp -Documenting Salt modules is easy! Just add a python docstring to the function. +The second and fourth lines are the start of the State Declarations, so they +are using the pkg and service states respectively. The pkg state manages a +software package to get installed via the system\(aqs native package manager, +and the service state manages a system daemon. Below the pkg and service +lines are the function to run. This function defines what state the named +package and service should be in. Here the package is to be installed, and +the service should be running. +.sp +Finally, on line 6, is the word \fBrequire\fP. This is called a Requisite +Statement, and it makes sure that the Apache service is only started after +the successful installation of the apache package. +.SS Adding Configs and Users +.sp +When setting up a service like an apache web server, many more components may +need to be added. The apache configuration file will most likely be managed, +and a user and group may need to be set up. .sp .nf .ft C -def spam(eggs): - \(aq\(aq\(aq - A function to make some spam with eggs! + apache: + pkg: + \- installed + service: + \- running + \- watch: + \- pkg: apache + \- file: /etc/httpd/conf/httpd.conf + \- user: apache + user: + \- present + \- uid: 87 + \- gid: 87 + \- home: /var/www/html + \- shell: /bin/nologin + \- require: + \- group: apache + group: + \- present + \- gid: 87 + \- require: + \- pkg: apache - CLI Example: - salt \(aq*\(aq test.spam eggs - \(aq\(aq\(aq - return eggs + /etc/httpd/conf/httpd.conf: + file: + \- managed + \- source: salt://apache/httpd.conf + \- user: root + \- group: root + \- mode: 644 +.ft P +.fi +.sp +This SLS data greatly extends the first example, and includes a config file, +a user, a group and new requisite statement: \fBwatch\fP. +.sp +Adding more states is easy, since the new user and group states are under +the apache ID, the user and group will be the apache user and group. The +\fBrequire\fP statements will make sure that the user will only be made after +the group, and that the group will be made only after the apache package is +installed. +.sp +Next,the \fBrequire\fP statement under service was changed to watch, and is +now watching 3 states instead of just one. The watch statement does the same +thing as require, making sure that the other states run before running the +state with a watch, but it adds an extra component. The \fBwatch\fP statement +will run the state\(aqs watcher function if any of the watched states changed +anything. So if the package was updated, the config file changed, or the user +uid modified, then the service state\(aqs watcher will be run. The service +state\(aqs watcher just restarts the service, so in this case, a change in the +config file will also trigger a restart of the respective service. +.SS Moving Beyond a Single SLS +.sp +When setting up Salt States, more than one SLS will need to be used. The above +examples were just in a single SLS file, but more than one SLS file can be +combined to build out a State Tree. The above example also references a file +with a strange source \- \fBsalt://apache/httpd.conf\fP. That file will need to +be available as well. +.sp +The SLS files are laid out in a directory on the salt master. Files are laid +out as just files. A sls is just a file and files to download are just files. +.sp +The apache example would be laid out in the root of the salt file server like +this: +.sp +.nf +.ft C +/apache/init.sls +/apache/httpd.conf .ft P .fi .sp -Now when the sys.doc call is executed the docstring will be cleanly returned -to the calling terminal. -.SS How Functions are Read +So the httpd.conf is just a file in the apache directory, and is referenced +directly. .sp -In Salt Python callable objects contained within a module are made available to -the Salt minion for use. The only exception to this rule is a callable object -with a name starting with an underscore \fB_\fP. -.SS Objects Loaded Into the Salt Minion +But with more than a single SLS file, more components can be added to the +toolkit, consider this ssh example: +.sp +\fB/ssh/init.sls:\fP .sp .nf .ft C -def foo(bar): - return bar + openssh\-client: + pkg: + \- installed -class baz: - def __init__(self, quo): - return quo + /etc/ssh/ssh_config + file: + \- managed + \- user: root + \- group: root + \- mode: 644 + \- source: salt://ssh/ssh_config + \- require: + \- pkg: openssh\-client .ft P .fi -.SS Objects NOT Loaded into the Salt Minion +.sp +\fBssh/server.sls:\fP .sp .nf .ft C -def _foobar(baz): # Preceded with an _ - return baz + include: + \- ssh -cheese = {} # Not a callable python object + openssh\-server: + pkg: + \- installed + + sshd: + service: + \- running + \- require: + \- pkg: openssh\-client + \- pkg: openssh\-server + \- file: /etc/ssh/banner + \- file: /etc/ssh/sshd_config + + /etc/ssh/sshd_config: + \- managed + \- user: root + \- group: root + \- mode: 644 + \- source: salt://ssh/sshd_config + \- require: + \- pkg: openssh\-server + + /etc/ssh/banner: + file: + \- managed + \- user: root + \- group: root + \- mode: 644 + \- source: salt://ssh/banner + \- require: + \- pkg: openssh\-server .ft P .fi -.SS Examples of Salt Modules .sp -The existing Salt modules should be fairly easy to read and understand, the -goal of the main distribution\(aqs Salt modules is not only to build a set of -functions for salt, but to stand as examples for building out more Salt -modules. +Now our State Tree looks like this: .sp -The existing modules can be found here: -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/modules\fP +.nf +.ft C +/apache/init.sls +/apache/httpd.conf +/ssh/init.sls +/ssh/server.sls +/ssh/banner +/ssh/ssh_config +/ssh/sshd_config +.ft P +.fi .sp -The most simple module is the test module, it contains the simplest salt -function, test.ping: +This example now introduces the \fBinclude\fP statement. The include statement +includes another SLS file so that components found in it can be required, +watched or as will soon be demonstrated \- extended. +.sp +The include statement allows for states to be cross linked. When an SLS +has an include statement it is literally extended to include the contents of +the included SLS files. +.SS Extending Included SLS Data +.sp +Sometimes SLS data needs to be extended. Perhaps the apache service needs to +watch additional resources, or under certain circumstances a different file +needs to be placed. +.sp +These examples will add more watchers to apache and change the ssh banner. +.sp +\fB/ssh/custom\-server.sls:\fP .sp .nf .ft C -def ping(): - \(aq\(aq\(aq - Just used to make sure the minion is up and responding - Return True + include: + \- ssh.server - CLI Example: - salt \(aq*\(aq test.ping - \(aq\(aq\(aq - return True + extend: + /etc/ssh/banner: + file: + \- source: salt://ssh/custom\-banner .ft P .fi -.SH FULL LIST OF BUILTIN MODULES -.TS -center; -|l|l|. -_ -T{ -\fBapache\fP -T} T{ -Support for Apache -T} -_ -T{ -\fBapt\fP -T} T{ -Support for APT (Advanced Packaging Tool) -T} -_ -T{ -\fBarchive\fP -T} T{ -A module to wrap archive calls -T} -_ -T{ -\fBbutterkvm\fP -T} T{ -Specialized routines used by the butter cloud component -T} -_ -T{ -\fBcluster\fP -T} T{ -The cluster module is used to distribute and activate salt HA cluster -T} -_ -T{ -\fBcmd\fP -T} T{ -A module for shelling out -T} -_ -T{ -\fBcp\fP -T} T{ -Minion side functions for salt\-cp -T} -_ -T{ -\fBcron\fP -T} T{ -Work with cron -T} -_ -T{ -\fBdata\fP -T} T{ -Manage a local persistent data structure that can hold any arbitrairy data -T} -_ -T{ -\fBdisk\fP -T} T{ -Module for gathering disk information -T} -_ -T{ -\fBebuild\fP -T} T{ -Support for Portage -T} -_ -T{ -\fBfile\fP -T} T{ -Manage information about files on the minion, set/read user, group, and mode -T} -_ -T{ -\fBfreebsdkmod\fP -T} T{ -Module to manage FreeBSD kernel modules -T} -_ -T{ -\fBfreebsdpkg\fP -T} T{ -Package support for FreeBSD -T} -_ -T{ -\fBgentoo_service\fP -T} T{ -Top level package command wrapper, used to translate the os detected by the -T} -_ -T{ -\fBgrains\fP -T} T{ -Control aspects of the grains data -T} -_ -T{ -\fBgroupadd\fP -T} T{ -Manage groups on Linux -T} -_ -T{ -\fBhosts\fP -T} T{ -Manage the information in the hosts file -T} -_ -T{ -\fBkmod\fP -T} T{ -Module to manage Linux kernel modules -T} -_ -T{ -\fBlinux_sysctl\fP -T} T{ -Module for viewing and modifying sysctl parameters -T} -_ -T{ -\fBmdadm\fP -T} T{ -Salt module to manage RAID arrays with mdadm -T} -_ -T{ -\fBmoosefs\fP -T} T{ -Module for gathering and managing information about MooseFS -T} -_ -T{ -\fBmount\fP -T} T{ -Salt module to manage unix mounts and the fstab file -T} -_ -T{ -\fBmysql\fP -T} T{ -Module to provide MySQL compatibility to salt. -T} -_ -T{ -\fBnetwork\fP -T} T{ -Module for gathering and managing network information -T} -_ -T{ -\fBpacman\fP -T} T{ -A module to wrap pacman calls, since Arch is the best -T} -_ -T{ -\fBpip\fP -T} T{ -Install Python packages with pip to either the system or a virtualenv -T} -_ -T{ -\fBpkg\fP -T} T{ -T} -_ -T{ -\fBps\fP -T} T{ -A salt interface to psutil, a system and process library. -T} -_ -T{ -\fBpublish\fP -T} T{ -Publish a command from a minion to a target -T} -_ -T{ -\fBpuppet\fP -T} T{ -Execute puppet routines -T} -_ -T{ -\fBpw_group\fP -T} T{ -Manage groups on Linux -T} -_ -T{ -\fBpw_user\fP -T} T{ -Manage users with the useradd command -T} -_ -T{ -\fBrh_service\fP -T} T{ -Top level package command wrapper, used to translate the os detected by the -T} -_ -T{ -\fBsaltutil\fP -T} T{ -The Saltutil module is used to manage the state of the salt minion itself. It is -T} -_ -T{ -\fBselinux\fP -T} T{ -Execute calls on selinux -T} -_ +.sp +\fB/python/mod_python.sls:\fP +.sp +.nf +.ft C + include: + \- apache + + extend: + apache: + service: + \- watch: + \- pkg: mod_python + + mod_python: + pkg: + \- installed +.ft P +.fi +.sp +The \fBcustom\-server.sls\fP file uses the extend statement to overwrite where the +banner is being downloaded from, and therefore changing what file is being used +to configure the banner. +.sp +In the new mod_python SLS the mod_python package is added, but more importantly +the apache service was extended to also watch the mod_python package. +.sp +There is a bit of a trick here, in the extend statement Requisite Statements +are extended, so the \fB\- pkg: mod_python\fP is appended to the watch list. But +all other statements are overwritten. +.SS Understanding the Render System +.sp +Since the SLS data is just plain old data, it does not need to be represented +with YAML. Salt defaults to YAML because it is very straightforward and easy +to learn and use. But the SLS files can be rendered from almost any imaginable +medium, so long as a renderer module is provided. +.sp +The default rendering system is the \fByaml_jinja\fP renderer. The +\fByaml_jinja\fP renderer will first pass the template through the jinja +templating system, and then through the YAML parser. The benefit here is that +full programming constructs are available when creating SLS files. +.sp +Other renderers available are \fByaml_mako\fP which uses the mako templating +system rather than the jinja templating system, and more notably, the pure +python or \fBpy\fP renderer. The \fBpy\fP renderer allows for SLS files to be +written in pure python, allowing for the utmost level of flexibility and +power when preparing SLS data. +.SS Getting to Know the Default \- yaml_jinja +.sp +The default renderer \- \fByaml_jinja\fP, allows for the use of the jinja +templating system. A guide to the jinja templating system can be found here: +\fI\%http://jinja.pocoo.org/docs\fP +.sp +When working with renderers a few very useful bits of data are passed in. In +the case of templating engine based renderers two critical components are +available, \fBsalt\fP, \fBgrains\fP, and \fBpillar\fP. The salt object allows for +any salt function to be called from within the template, and grains allows for +the grains to be accessed from within the template. A few examples: +.sp +\fB/apache/init.sls:\fP +.sp +.nf +.ft C + apache: + pkg: + \- installed + {% if grains[\(aqos\(aq] == \(aqRedHat\(aq%} + \- name: httpd + {% endif %} + service: + \- running + {% if grains[\(aqos\(aq] == \(aqRedHat\(aq%} + \- name: httpd + {% endif %} + \- watch: + \- pkg: apache + \- file: /etc/httpd/conf/httpd.conf + \- user: apache + user: + \- present + \- uid: 87 + \- gid: 87 + \- home: /var/www/html + \- shell: /bin/nologin + \- require: + \- group: apache + group: + \- present + \- gid: 87 + \- require: + \- pkg: apache + + /etc/httpd/conf/httpd.conf: + file: + \- managed + \- source: salt://apache/httpd.conf + \- user: root + \- group: root + \- mode: 644 +.ft P +.fi +.sp +This example is simple. If the \fBos\fP grain states that the operating system is +Red Hat, then the name of the apache package and service needs to be httpd. +.sp +A more aggressive way to use Jinja can be found here, in a module to set up +a MooseFS distributed filesystem chunkserver: +.sp +\fB/moosefs/chunk.sls:\fP +.sp +.nf +.ft C + include: + \- moosefs + + {% for mnt in salt[\(aqcmd.run\(aq](\(aqls /dev/data/moose*\(aq).split() %} + /mnt/moose{{ mnt[\-1] }}: + mount: + \- mounted + \- device: {{ mnt }} + \- fstype: xfs + \- mkmnt: True + file: + \- directory + \- user: mfs + \- group: mfs + \- require: + \- user: mfs + \- group: mfs + {% endfor %} + + \(aq/etc/mfshdd.cfg\(aq: + file: + \- managed + \- source: salt://moosefs/mfshdd.cfg + \- user: root + \- group: root + \- mode: 644 + \- template: jinja + \- require: + \- pkg: mfs\-chunkserver + + \(aq/etc/mfschunkserver.cfg\(aq: + file: + \- managed + \- source: salt://moosefs/mfschunkserver.cfg + \- user: root + \- group: root + \- mode: 644 + \- template: jinja + \- require: + \- pkg: mfs\-chunkserver + + mfs\-chunkserver: + pkg: + \- installed + mfschunkserver: + service: + \- running + \- require: + {% for mnt in salt[\(aqcmd.run\(aq](\(aqls /dev/data/moose*\(aq) %} + \- mount: /mnt/moose{{ mnt[\-1] }} + \- file: /mnt/moose{{ mnt[\-1] }} + {% endfor %} + \- file: /etc/mfschunkserver.cfg + \- file: /etc/mfshdd.cfg + \- file: /var/lib/mfs +.ft P +.fi +.sp +This example shows much more of the available power provided by Jinja. +Multiple for loops are used to dynamically detect available hard drives +and set them up to be mounted, and the \fBsalt\fP object is used multiple +times to call shell commands to gather data. +.SS Introducing the Python Renderer +.sp +Sometimes the chosen default renderer might not have enough logical power to +accomplish the needed task. When this happens, the python renderer can be +used. Normally a yaml renderer should be used for the majority of SLS files, +but a SLS file set to use another renderer can be easily added to the tree. +.sp +This example shows a very basic python SLS file: +.sp +\fB/python/django.sls:\fP +.sp +.nf +.ft C + #!py + + def run(): + \(aq\(aq\(aq + Install the django package + \(aq\(aq\(aq + return {\(aqinclude\(aq: [\(aqpython\(aq], + \(aqdjango\(aq: {\(aqpkg\(aq: [\(aqinstalled\(aq]}} +.ft P +.fi +.sp +This is a very simple example, the first line has a SLS shebang line that +tells Salt to not use the default renderer, but to use the \fBpy\fP renderer. +Then the run function is defined, the return value from the run function +must be a Salt friendly data structure, or better known as a Salt +\fBHighState data structure\fP. +.sp +This python example would look like this if it were written in YAML: +.sp +.nf +.ft C + include: + \- python + + django: + pkg: + \- installed +.ft P +.fi +.sp +This clearly illustrates, that not only is using the YAML renderer a wise +decision as the default, but that unbridled power can be obtained where +needed by using a pure python SLS. +.sp +Now onto the \fBStates tutorial, part 1\fP. +.SH STATES TUTORIAL, PART 1 +.sp +The purpose of this tutorial is to demonstrate how quickly you can configure a +system to be managed by Salt States. For detailed information about the state +system please refer to the full \fBstates reference\fP. +.sp +This tutorial will walk you through using Salt to configure a minion to run the +Apache HTTP server and to ensure the server is running. +.sp +\fBBefore continuing\fP make sure you have a working Salt installation by +following the \fI\%installation\fP and the \fBconfiguration\fP instructions. +.IP "Stuck?" +.sp +If you get stuck at any point, there are many ways to \fBget help from +the Salt community\fP including our mailing list and our +IRC channel. +.RE +.SS Setting up the Salt State Tree +.sp +States are stored in text files on the master and transfered to the minions on +demand via the master\(aqs File Server. The collection of state files make up the +\fIState Tree\fP. +.sp +To start using a central state system in Salt you must first set up the Salt +File Server. Edit your master config file (\fBfile_roots\fP) and +uncomment the following lines: +.sp +.nf +.ft C +file_roots: + base: + \- /srv/salt +.ft P +.fi +.IP Note +If you are deploying on FreeBSD via ports, the \fBfile_roots\fP path defaults +to \fB/usr/local/etc/salt/states\fP. +.RE +.sp +Restart the Salt master in order to pick up this change: +.sp +.nf +.ft C +% pkill salt\-master +% salt\-master \-d +.ft P +.fi +.SS Preparing the Top File +.sp +On the master in the directory you specified in the previous step, create a new +file called \fBtop.sls\fP and add the following: +.sp +.nf +.ft C +base: + \(aq*\(aq: + \- webserver +.ft P +.fi +.sp +The \fItop file\fP is separated into environments (discussed later). The +default environment is \fBbase\fP. Under the \fBbase\fP environment a collection of +minion matches is defined; for now simply specify all hosts (\fB*\fP). +.IP "Targeting minions" +.sp +The expressions can use any of the targeting mechanisms used by Salt — +minions can be matched by glob, pcre regular expression, or by \fBgrains\fP. For example: +.sp +.nf +.ft C +base: + \(aqos:Fedora\(aq: + \- match: grain + \- webserver +.ft P +.fi +.RE +.SS Create an \fBsls\fP module +.sp +In the same directory as your \fItop file\fP, create an empty file, called an +\fIsls module\fP, named \fBwebserver.sls\fP. Type the following and save the +file: +.sp +.nf +.ft C +apache: # ID declaration + pkg: # state declaration + \- installed # function declaration +.ft P +.fi +.sp +The first line, called the \fIID declaration\fP, is an arbitrary identifier. +In this case it defines the name of the package to be installed. \fBNOTE:\fP the +package name for the Apache httpd web server may differ on your OS or distro — +for example, on Fedora it is \fBhttpd\fP but on Debian/Ubuntu it is \fBapache2\fP. +.sp +Additionally, an ID declaration should not contain a dot, as this will produce +unpredictable output in the summary returned from a call to +\fBstate.highstate\fP. +.sp +The second line, called the \fIstate declaration\fP, defines which of the +Salt States we are using. In this example, we are using the \fBpkg state\fP to ensure that a given package is installed. +.sp +The third line, called the \fIfunction declaration\fP, defines which function +in the \fBpkg state\fP module to call. +.IP "Renderers" +.sp +States \fIsls\fP files can be written in many formats. Salt requires only +a simple data structure and is not concerned with how that data structure +is built. Templating languages and \fI\%DSLs\fP are a dime\-a\-dozen and everyone +has a favorite. +.sp +Building the expected data structure is the job of Salt \fBrenderers\fP and they are dead\-simple to write. +.sp +In this tutorial we will be using YAML in Jinja2 templates, which is the +default format. You can change the default by changing +\fBrenderer\fP in the master configuration file. +.RE +.SS Install the package +.sp +Next, let\(aqs run the state we created. Open a terminal on the master and run: +.sp +.nf +.ft C +% salt \(aq*\(aq state.highstate +.ft P +.fi +.sp +Our master is instructing all targeted minions to run \fBstate.highstate\fP. When a minion executes a highstate call it +will download the \fItop file\fP and attempt to match the expressions. When +it does match an expression the modules listed for it will be downloaded, +compiled, and executed. +.sp +Once completed, the minion will report back with a summary of all actions taken +and all changes made. +.IP "Troubleshooting Salt" +.sp +In case you don\(aqt see the expected output, the following tips can help you +narrow down the problem. +.INDENT 0.0 +.TP +.B Turn up logging +Salt can be quite chatty when you change the logging setting to +\fBdebug\fP: +.sp +.nf +.ft C +salt\-minion \-l debug +.ft P +.fi +.TP +.B Run the minion in the foreground +By not starting the minion in daemon mode (\fI\-d\fP) you can view any output from the minion as it works: +.sp +.nf +.ft C +salt\-minion & +.ft P +.fi +.UNINDENT +.sp +Increase the default timeout value when running \fBsalt\fP. For +example, to change the default timeout to 60 seconds: +.sp +.nf +.ft C +salt \-t 60 +.ft P +.fi +.sp +For best results, combine all three: +.sp +.nf +.ft C +salt\-minion \-l debug & # On the minion +salt \(aq*\(aq state.highstate \-t 60 # On the master +.ft P +.fi +.RE +.SS Next steps +.sp +This tutorial focused on getting a simple Salt States configuration working. +\fBPart 2\fP will build on this example to cover more advanced +\fIsls\fP syntax and will explore more of the states that ship with Salt. +.SH STATES TUTORIAL, PART 2 +.sp +This tutorial builds on the topic covered in \fBpart 1\fP. It is +recommended that you begin there. +.sp +In the last Salt States tutorial we covered the basics of installing a package. +In this tutorial we will modify our \fBwebserver.sls\fP file to be more +complicated, have requirements, and use even more Salt States. +.SS Call multiple States +.sp +You can specify multiple \fIstate declarations\fP under +an \fIID declaration\fP. For example, a quick modification to our +\fBwebserver.sls\fP to also start Apache if it is not running: +.sp +.nf +.ft C +apache: + pkg: + \- installed + service: + \- running +.ft P +.fi +.sp +Try stopping Apache before running \fBstate.highstate\fP once again and observe +the output. +.SS Expand the SLS module +.sp +As you have seen, sls modules are appended with the file extension \fB.sls\fP and +are referenced by name starting at the root of the state tree. An SLS module +can be also defined as a directory. Demonstrate that now by creating a +directory named \fBwebserver\fP and moving and renaming \fBwebserver.sls\fP to +\fBwebserver/init.sls\fP. Your state directory should now resemble: +.sp +.nf +.ft C +|\- top.sls +\(ga\- webserver/ + \(ga\- init.sls +.ft P +.fi +.IP "Organizing SLS modules" +.sp +You can place additional \fB.sls\fP files in a state file directory. This +affords much cleaner organization of your state tree on the filesystem. For +example, if we created a \fBwebserver/django.sls\fP file that module would be +referenced as \fBwebserver.django\fP. +.sp +In addition, States provide powerful includes and extending functionality +which we will cover in \fBPart 3\fP. +.RE +.SS Require other states +.sp +We now have a working installation of Apache so let\(aqs add an HTML file to +customize our website. It isn\(aqt exactly useful to have a website without a +webserver so we don\(aqt want Salt to install our HTML file until Apache is +installed and running. Include the following at the bottom of your +\fBwebserver/init.sls\fP file: +.sp +.nf +.ft C +apache: + pkg: + \- installed + service: + \- running + +/var/www/index.html: # ID declaration + file: # state declaration + \- managed # function + \- source: salt://webserver/index.html # function arg + \- require: # requisite declaration + \- pkg: apache # requisite reference +.ft P +.fi +.sp +\fBline 7\fP is the \fIID declaration\fP. In this example it is the +location we want to install our custom HTML file. (\fBNote:\fP the default +location that Apache serves may differ from the above on your OS or distro. +\fB/srv/www\fP could also be a likely place to look.) +.sp +\fBLine 8\fP the \fIstate declaration\fP. This example uses the Salt \fBfile +state\fP. +.sp +\fBLine 9\fP is the \fIfunction declaration\fP. The \fBmanaged function\fP will download a file from the master and install it +in the location specified. +.sp +\fBLine 10\fP is a \fIfunction arg declaration\fP which, in this example, passes +the \fBsource\fP argument to the \fBmanaged function\fP. +.sp +\fBLine 11\fP is a \fIrequisite declaration\fP. +.sp +\fBLine 12\fP is a \fIrequisite reference\fP which refers to a state and an ID. +In this example, it is referring to the \fBID declaration\fP from our example in +\fBpart 1\fP. This declaration tells Salt not to install the HTML +file until Apache is installed. +.sp +Next, create the \fBindex.html\fP file and save it in the \fBwebserver\fP +directory: +.sp +.nf +.ft C +<html> + <head><title>Salt rocks</title></head> + <body> + <h1>This file brought to you by Salt</h1> + </body> +</html> +.ft P +.fi +.sp +Last, call \fBstate.highstate\fP again and the +minion will fetch and execute the highstate as well as our HTML file from the +master using Salt\(aqs File Server: +.sp +.nf +.ft C +salt \(aq*\(aq state.highstate +.ft P +.fi +.sp +Verify that Apache is now serving your custom HTML. +.IP "\fBrequire\fP vs. \fBwatch\fP" +.sp +There are two \fIrequisite declarations\fP, +“require” and “watch”. Not every state supports “watch”. The \fBservice +state\fP does support “watch” and will restart a +service based on the watch condition. +.sp +For example, if you use Salt to install an Apache virtual host +configuration file and want to restart Apache whenever that file is changed +you could modify our Apache example from earlier as follows: +.sp +.nf +.ft C +/etc/httpd/extra/httpd\-vhosts.conf: + file: + \- managed + \- source: salt://webserver/httpd\-vhosts.conf + +apache: + pkg: + \- installed + service: + \- running + \- watch: + \- file: /etc/httpd/extra/httpd\-vhosts.conf +.ft P +.fi +.sp +If the pkg and service names differ on your OS or distro of choice you can +specify each one separately using a \fIname declaration\fP which +explained in \fBPart 3\fP. +.RE +.SS Next steps +.sp +In \fBpart 3\fP we will discuss how to use includes, extends and +templating to make hugely complicated State Tree configurations dead\-simple. +.SH STATES TUTORIAL, PART 3 +.sp +This tutorial builds on the topic covered in \fBpart 2\fP. It is +recommended that you begin there. +.sp +This tutorial will cover more advanced templating and configuration techniques +for \fBsls\fP files. +.SS Templating SLS modules +.sp +SLS modules may require programming logic or inline executions. This is +accomplished with module templating. The default module templating system used +is \fI\%Jinja2\fP and may be configured by changing the \fBrenderer\fP +value in the master config. +.sp +All states are passed through a templating system when they are initially read, +so all that is required to make use of the templating system is to add some +templating code. An example of an sls module with templating may look like +this: +.sp +.nf +.ft C +{% for usr in \(aqmoe\(aq,\(aqlarry\(aq,\(aqcurly\(aq %} +{{ usr }}: + user: + \- present +{% endfor %} +.ft P +.fi +.sp +This templated sls file once generated will look like this: +.sp +.nf +.ft C +moe: + user: + \- present +larry: + user: + \- present +currly: + user: + \- present +.ft P +.fi +.SS Using Grains in SLS modules +.sp +Often times a state will need to behave differently on different systems. +\fBSalt grains\fP can be used from within sls modules. An object +called \fBgrains\fP is made available in the template context: +.sp +.nf +.ft C +apache: + pkg: + {% if grains[\(aqos\(aq] == \(aqRedHat\(aq %} + \- name: httpd + {% elif grains[\(aqos\(aq] == \(aqUbuntu\(aq %} + \- name: apache2 + {% endif %} + \- installed +.ft P +.fi +.SS Calling Salt modules from templates +.sp +All of the Salt modules loaded by the minion are available within the +templating system. This allows data to be gathered in real time on the target +system. It also allows for shell commands to be run easily from within the sls +modules. +.sp +The Salt module functions are also made available in the template context as +\fBsalt\fP: +.sp +.nf +.ft C +{% for usr in \(aqmoe\(aq,\(aqlarry\(aq,\(aqcurly\(aq %} +{{ usr }}: + group: + \- present + user: + \- present + \- gid: {{ salt[\(aqfile.group_to_gid\(aq](usr) }} + \- require: + \- group: {{ usr }} +{% endfor %} +.ft P +.fi +.sp +Below is an example that uses the \fBnetwork.hwaddr\fP function to retrieve the +MAC address for eth0: +.INDENT 0.0 +.INDENT 3.5 +salt[\(aqnetwork.hwaddr\(aq](\(aqeth0\(aq) +.UNINDENT +.UNINDENT +.SS Advanced SLS module syntax +.sp +Last we will cover some incredibly useful techniques for more complex State +trees. +.SS \fIInclude declaration\fP +.sp +You have seen an example of how to spread a Salt tree across several files but +in order to be able to have \fIrequisite references\fP +span multiple files you must use an \fIinclude declaration\fP. For example: +.sp +\fBpython\-libs.sls\fP: +.sp +.nf +.ft C +python\-dateutil: + pkg: + \- installed +.ft P +.fi +.sp +\fBdjango.sls\fP: +.sp +.nf +.ft C +include: + \- python\-libs + +django: + pkg: + \- installed + \- require: + \- pkg: python\-dateutil +.ft P +.fi +.SS \fIExtend declaration\fP +.sp +You can modify previous declarations by using an \fIextend declaration\fP. For +example the following modifies the Apache tree to also restart Apache when the +vhosts file is changed: +.sp +\fBapache.sls\fP: +.sp +.nf +.ft C +apache: + pkg: + \- installed +.ft P +.fi +.sp +\fBmywebsite.sls\fP: +.sp +.nf +.ft C +include: + \- apache + +extend: + apache: + service: + \- watch: + \- file: /etc/httpd/extra/httpd\-vhosts.conf + +/etc/httpd/extra/httpd\-vhosts.conf: + file: + \- managed + \- source: salt://httpd\-vhosts.conf +.ft P +.fi +.SS \fIName declaration\fP +.sp +You can override the \fIID declaration\fP by using a \fIname +declaration\fP. For example, the previous example is a bit more maintainable if +rewritten as follows: +.sp +\fBmywebsite.sls\fP: +.sp +.nf +.ft C +include: + \- apache + +extend: + apache + service: + \- watch: + \- file: mywebsite + +mywebsite: + file: + \- managed + \- name: /etc/httpd/extra/httpd\-vhosts.conf + \- source: salt://httpd\-vhosts.conf +.ft P +.fi +.SS \fINames declaration\fP +.sp +Even more powerful is using a \fInames declaration\fP to override the +\fIID declaration\fP for multiple states at once. This often can remove the +need for looping in a template. For example, the first example in this tutorial +can be rewritten without the loop: +.sp +.nf +.ft C +stooges: + user: + \- present + \- names: + \- moe + \- larry + \- curly +.ft P +.fi +.SS Continue learning +.sp +The best way to continue learning about Salt States is to read through the +\fBreference documentation\fP and to look through examples +of existing \fIstate trees\fP. You can find examples in the +\fI\%salt-states repository\fP and please send a pull\-request on GitHub with any +state trees that you build and want to share! +.sp +If you have any questions, suggestions, or just want to chat with other people +who are using Salt we have an \fBactive community\fP. +.SH OPENING THE FIREWALL UP FOR SALT +.sp +The Salt master communicates with the minions using an AES\-encrypted ZeroMQ +connection. These communications are done over ports 4505 and 4506, which need +to be accessible on the master only. This document outlines suggested firewall +rules for allowing these incoming connections to the master. +.IP Note +\fBNo firewall configuration needs to be done on Salt minions. These changes +refer to the master only.\fP +.RE +.SS iptables +.sp +Different Linux distributions store their iptables rules in different places, +which makes it difficult to standardize firewall documentation. I\(aqve included +some of the more common locations, but your mileage may vary. +.sp +\fBFedora / Red Hat / CentOS\fP +.sp +.nf +.ft C +/etc/sysconfig/iptables +.ft P +.fi +.sp +\fBArch Linux\fP +.sp +.nf +.ft C +/etc/iptables/iptables.rules +.ft P +.fi +.sp +\fBDebian\fP +.sp +Follow these instructions: \fI\%http://wiki.debian.org/iptables\fP +.sp +Once you\(aqve found your firewall rules, you\(aqll need to add the two lines below +to allow traffic on \fBtcp/4505\fP and \fBtcp/4506\fP: +.sp +.nf +.ft C ++ \-A INPUT \-m state \-\-state new \-m tcp \-p tcp \-\-dport 4505 \-j ACCEPT ++ \-A INPUT \-m state \-\-state new \-m tcp \-p tcp \-\-dport 4506 \-j ACCEPT +.ft P +.fi +.sp +\fBUbuntu\fP +.sp +Create a file named \fB/etc/ufw/applications.d/salt\-master\fP +.sp +.nf +.ft C +[Salt Master] +title=Salt master +description=Salt is a remote execution and configuration management tool. +ports=4205,4206/tcp +.ft P +.fi +.SS pf.conf +.sp +The BSD\-family of operating systems uses packet filter (pf). The following +example describes the additions to \fBpf.conf\fP needed to access the Salt +master. +.sp +.nf +.ft C ++ pass in on $int_if proto tcp from any to $int_if port 4505 ++ pass in on $int_if proto tcp from any to $int_if port 4506 +.ft P +.fi +.sp +Once you\(aqve made these additions to your \fBpf.conf\fP you\(aqll need to reload the +new rules with the new additions. This can be done using the \fBpfctl\fP command. +.sp +.nf +.ft C +pfctl \-vf /etc/pf.conf +.ft P +.fi +.SH BOOSTRAPPING SALT ON LINUX EC2 WITH CLOUD-INIT +.sp +\fI\%Salt\fP is a great tool for remote execution and +configuration management, however you will still need to bootstrap the +daemon when spinning up a new node. One option is to create and save a +custom AMI, but this creates another resource to maintain and document. +.sp +A better method for Linux machines uses Canonical\(aqs \fI\%CloudInit\fP to run a bootstrap script +during an EC2 Instance initialization. Cloud\-init takes the \fBuser_data\fP +string passed into a new AWS instance and runs it in a manner similar to +rc.local. The bootstrap script needs to: +.INDENT 0.0 +.IP 1. 3 +Install \fI\%Salt\fP with dependencies +.IP 2. 3 +Point the minion to the master +.UNINDENT +.sp +Here is a sample script: +.sp +.nf +.ft C +#!/bin/bash + +# Install saltstack +add\-apt\-repository ppa:saltstack/salt \-y +apt\-get update \-y +apt\-get install salt \-y +apt\-get upgrade \-y + +# Set salt master location and start minion +cp /etc/salt/minion.template /etc/salt/minion +sed \-i \(aq\(aq \-e \(aqs/#master: salt/master: [salt_master_fqdn]\(aq /etc/salt/minion +salt\-minion \-d +.ft P +.fi +.sp +First the script adds the saltstack ppa and installs the package. Then +we copy over the minion config template and tell it where to find the +master. You will have to replace \fB[salt_master_fqdn]\fP with something +that resolves to your salt master. +.SS Used With Boto +.sp +\fI\%Boto\fP will accept a string for user data +which can be used to pass our bootstrap script. If the script is saved to +a file, you can read it into a string: +.sp +.nf +.ft C +import boto + +user_data = open(\(aqsalt_bootstrap.sh\(aq) + +conn = boto.connect_ec2(<AWS_ACCESS_ID>, <AWS_SECRET_KEY>) + +reservation = conn.run_instances(image_id=<ami_id>, + key_name=<key_name>, + user_data=user_data.read()) +.ft P +.fi +.SS Additional Notes +.sp +Sometime in the future the ppa will include and install an upstart file. In the meantime, you can use the bootstrap to \fI\%build one\fP. +.sp +It may also be useful to set the node\(aqs role during this phase. One option +would be saving the node\(aqs role to a file and then using a custom grain +to select it. +.SH PILLAR OF SALT +.sp +Pillar is an interface for Salt designed to offer global values to be +distributed to all minions. Pillar data is managed in a similar way to +the salt state tree. +.sp +Pillar was added to Salt in version 0.9.8 as an experimental add on. +.SS Declaring the Master Pillar +.sp +The Salt Master server maintains a pillar_roots setup that matches the +structure of the file_roots used in the Salt file server. Like the +Salt file server the \fBpillar_roots\fP option in the master config is based +on environments mapping to directories. The pillar data is then mapped to +minions based on matchers in a top file which is laid out in the same way +as the state top file. +.sp +The configuration for the pillar_roots in the master config is identical in +behavior and function as the file_roots configuration: +.sp +.nf +.ft C +pillar_roots: + base: + \- /srv/pillar +.ft P +.fi +.sp +This example configuration declares that the base environment will be located +in the /srv/pillar directory. The top file used matches the name of the top file +used for states, and has the same structure: +.sp +.nf +.ft C +base: + \(aq*\(aq: + \- packages +.ft P +.fi +.sp +This simple pillar top file declares that information for all minions can be +found in the package\(aqs sls file: +.sp +.nf +.ft C +{% if grains[\(aqos\(aq] == \(aqRedHat\(aq %} +apache: httpd +git: git +{% elif grains[\(aqos\(aq] == \(aqDebian\(aq %} +apache: apache2 +git: git\-core +{% endif %} +.ft P +.fi +.sp +Now this data can be used from within modules, renderers, state sls files and +more via the shared pillar dict: +.sp +.nf +.ft C +apache: + pkg: + \- installed + \- name: {{ pillar[\(aqapache\(aq] }} +.ft P +.fi +.sp +.nf +.ft C +git: + pkg: + \- installed + \- name: {{ pillar[\(aqgit\(aq] }} +.ft P +.fi +.SH JOB MANAGEMENT +.sp +New in version 0.9.7. +.sp +Since Salt executes jobs running on many systems, Salt needs to be able to +manage jobs running on many systems. As of Salt 0.9.7 the capability was +added for more advanced job management. +.SS The Minion proc System +.sp +The Salt Minions now maintain a proc directory in the salt cachedir, the proc +directory maintains files named after the executed job id. These files contain +the information about the current running jobs on the minion and allow for +jobs to be looked up. This is located in the proc directory under the +cachedir, with a default configuration it is under /var/cache/salt/proc. +.SS Functions in the saltutil Module +.sp +Salt 0.9.7 introduced a few new functions to the +\fBsaltutil\fP module for managing +jobs. These functions are: +.INDENT 0.0 +.IP 1. 3 +\fBrunning\fP +Returns the data of all running jobs that are found in the proc directory. +.IP 2. 3 +\fBfind_job\fP +Returns specific data about a certain job based on job id. +.IP 3. 3 +\fBsignal_job\fP +Allows for a given jid to be sent a signal. +.IP 4. 3 +\fBterm_job\fP +Sends a termination signal (SIGTERM, 15) to the process controlling the +specified job. +.IP 5. 3 +\fBkill_job\fP +Sends a kill signal (SIGKILL, 9) to the process controlling the +specified job. +.UNINDENT +.sp +These functions make up the core of the back end used to manage jobs at the +minion level. +.SS The jobs Runner +.sp +A convenience runner front end and reporting system has been added as well. +The jobs runner contains functions to make viewing data easier and cleaner. +.sp +The jobs runner contains a number of functions... +.SS active +.sp +The active function runs saltutil.running on all minions and formats the +return data about all running jobs in a much more usable and compact format. +The active function will also compare jobs that have returned and jobs that +are still running, making it easier to see what systems have completed a job +and what systems are still being waited on. +.SS lookup_jid +.sp +When jobs are executed the return data is sent back to the master and cached. +By default is is cached for 24 hours, but this can be configured via the +\fBkeep_jobs\fP option in the master configuration. +Using the lookup_jid runner will display the same return data that the initial +job invocation with the salt command would display. +.SS list_jobs +.sp +Before finding a historic job, it may be required to find the job id. list_jobs +will parse the cached execution data and display all of the job data for jobs +that have already, or partially returned. +.SH TROUBLESHOOTING +.sp +The intent of the troubleshooting section is to introduce solutions to a +number of common issues encountered by users and the tools that are available +to aid in developing states and salt code. +.SS Running in the Foreground +.sp +A great deal of information is available via the debug logging system, if you +are having issues with minions connecting or not starting run the minion and/or +master in the foreground: +.sp +.nf +.ft C +# salt\-master \-l debug +# salt\-minion \-l debug +.ft P +.fi +.SS What Ports do the Master and Minion Need Open? +.sp +No ports need to be opened up on each minion. For the master, tcp ports 4505 +and 4506 need to be open. If you\(aqve put your salt master and minion both in +debug mode and don\(aqt see an acknowledgement that your minion has connected, +it could very well be a firewall. +.sp +You can check port connectivity from the minion with the nc command: +.sp +.nf +.ft C +# nc \-v \-z salt.master.ip 4505 +# nc \-v \-z salt.master.ip 4506 +.ft P +.fi +.sp +There is also a \fBfirewall configuration\fP +document that might help as well. +.SS Using salt\-call +.sp +The \fBsalt\-call\fP command was originally developed for aiding in the development +of new salt modules. Since then many applications have arisen for the salt\-call +command that is bundled with the salt minion. These range from the original +intent of the salt\-call, development assistance, to gathering large amounts of +data from complex calls like +\fBstate.highstate\fP. +.sp +When developing the state tree it is generally recommended to invoke +state.highstate with salt\-call, this displays a great deal more information +about the highstate execution than if it is called remotely. +.SS Too many open files +.sp +The salt\-master needs at least 2 sockets per host that connects to it, one for +the Publisher and one for response port. Thus, large installations may upon +scaling up the number of minions accessing a given master, encounter: +.sp +.nf +.ft C +12:45:29,289 [salt.master ][INFO ] Starting Salt worker process 38 +Too many open files +sock != \-1 (tcp_listener.cpp:335) +.ft P +.fi +.sp +The solution to this would be to check the number of files allowed to be +opened by the user running salt\-master (root by default): +.sp +.nf +.ft C +[root@salt\-master ~]# ulimit \-n +1024 +.ft P +.fi +.sp +And modify that value to be at least equal to the number of minions x 2. +This setting can be changed in limits.conf as the nofile value(s), +and activated upon new a login of the specified user. +.sp +So, an environment with 1800 minions, would need 1800 x 2 = 3600 as a minimum. +.SS Salt Master Stops Responding +.sp +There are known bugs with ZeroMQ less than 2.1.11 which can cause the salt +master to not respond properly. If you\(aqre running ZeroMQ greater than or equal +to 2.1.9, you can work around the bug by setting the sysctls +\fBnet.core.rmem_max\fP and \fBnet.core.wmem_max\fP to 16777216. Next set the third +field in \fBnet.ipv4.tcp_rmem\fP and \fBnet.ipv4.tcp_wmem\fP to at least 16777216. +.sp +You can do it manually with something like: +.sp +.nf +.ft C +# echo 16777216 > /proc/sys/net/core/rmem_max +# echo 16777216 > /proc/sys/net/core/wmem_max +# echo "4096 87380 16777216" > /proc/sys/net/ipv4/tcp_rmem +# echo "4096 87380 16777216" > /proc/sys/net/ipv4/tcp_wmem +.ft P +.fi +.sp +Or with the following salt state: +.sp +.nf +.ft C +net.core.rmem_max: + sysctl: + \- present + \- value: 16777216 + +net.core.wmem_max: + sysctl: + \- present + \- value: 16777216 + +net.ipv4.tcp_rmem: + sysctl: + \- present + \- value: 4096 87380 16777216 + +net.ipv4.tcp_wmem: + sysctl: + \- present + \- value: 4096 87380 16777216 +.ft P +.fi +.SS Common YAML Gotchas +.sp +An extensive list of +\fByaml idiosyncrasies\fP +has been compiled. +.SH YAML IDIOSYNCRASIES +.sp +One of Salt\(aqs strengths, the use of existing serialization systems for +representing sls data, can also backfire. YAML is a general purpose system +and there are a number of things that would seem to make sense in an sls +file that cause YAML issues. It is wise to be aware of these issues. While +reports or running into them are generally rare they can still crop up at +unexpected times. +.SS Spaces vs Tabs +.sp +Yaml uses spaces, period. Do not use tabs in your sls files! If strange +errors are coming up in rendering sls files, make sure to check that +no tabs have crept in! In vi / vim, you can check with \fB:se spell\fP. +.SS Indentation +.sp +The suggested syntax for YAML files is to use 2 spaces for indentation, +but YAML will follow whatever indentation system that the individual file +uses. Indentation of two spaces works very well for sls files given the +fact that the data is uniform and not deeply nested. +.SS Nested Dicts (key=value) +.sp +When dicts are more deeply nested they no longer follow the same indentation +logic. This is rarely something that comes up in Salt, since deeply nested +options like these are discouraged when making state modules, but some do +exist. A good example is the context and default options in the +\fBfile.managed\fP state: +.sp +.nf +.ft C +/etc/http/conf/http.conf: + file: + \- managed + \- source: salt://apache/http.conf + \- user: root + \- group: root + \- mode: 644 + \- template: jinja + \- context: + custom_var: "override" + \- defaults: + custom_var: "default value" + other_var: 123 +.ft P +.fi +.sp +Notice that the spacing used is 2 spaces, and that when defining the context +and defaults options there is a 4 space indent. If only a 2 space indent is +used then the information will not be loaded correctly. If using double spacing +is not desirable, then a deeply nested dict can be declared with curly braces: +.sp +.nf +.ft C +/etc/http/conf/http.conf: + file: + \- managed + \- source: salt://apache/http.conf + \- user: root + \- group: root + \- mode: 644 + \- template: jinja + \- context: { + custom_var: "override" } + \- defaults: { + custom_var: "default value" + other_var: 123 } +.ft P +.fi +.SS Integers are Parsed as Integers +.sp +When passing integers into an sls file, they are passed as integers. This means +that if a state accepts a string value and an integer is passed, that an +integer will be sent. The solution here is to send the integer as a string. +.sp +This is best explained when setting the mode for a file: +.sp +.nf +.ft C +/etc/vimrc: + file: + \- managed + \- source: salt://edit/vimrc + \- user: root + \- group: root + \- mode: 644 +.ft P +.fi +.sp +Salt manages this well, since the mode is passed as 644, but if the mode is +zero padded as 0644, then it is read by YAML as an integer and evaluated as +a hexadecimal value, 0644 becomes 420. Therefore, if the file mode is +preceded by a 0 then it needs to be passed as a string: +.sp +.nf +.ft C +/etc/vimrc: + file: + \- managed + \- source: salt://edit/vimrc + \- user: root + \- group: root + \- mode: \(aq0644\(aq +.ft P +.fi +.SH COMMUNITY +.sp +Join the Salt! +.sp +There are many ways to participate in and communicate with the Salt community. +.sp +Salt has an active IRC channel and a mailing list. +.SS Mailing List +.sp +Join the \fI\%salt-users mailing list\fP. It is the best place to ask questions +about Salt and see whats going on with Salt development! The Salt mailing list +is hosted by Google Groups. It is open to new members. +.sp +\fI\%http://groups.google.com/group/salt-users\fP +.SS IRC +.sp +The \fB#salt\fP IRC channel is hosted on the popular \fI\%Freenode\fP network. You +can use the \fI\%Freenode webchat client\fP right from your browser. +.sp +\fI\%Logs of the IRC channel activity\fP are being collected courtesy of Moritz Lenz. +.SS Follow on Github +.sp +The Salt code is developed via Github. Follow Salt for constant updates on what +is happening in Salt development: +.sp +\fI\%https://github.com/saltstack/salt\fP +.SS The Red45 Blog +.sp +News and thoughts on Salt and related projects is often posted on Thomas\(aq blog +\fI\%The Red45\fP: +.sp +\fI\%http://red45.wordpress.com/\fP +.SS Example Salt States +.sp +The official \fBsalt\-states\fP repository is: +\fI\%https://github.com/saltstack/salt-states\fP +.sp +Another good example from one of our users is: +\fI\%https://github.com/blast-hardcheese/blast-salt-states\fP +.SS Follow on ohloh +.sp +\fI\%https://www.ohloh.net/p/salt\fP +.SS Developing Salt +.sp +If you want to help develop Salt there is a great need and your patches are +welcome! +.sp +To assist in Salt development, you can help in a number of ways. +.SS Posting patches to the mailing list +.sp +If you have a patch for Salt, please format it via \fBgit format\-patch\fP and +send it to the Salt users mailing list. This allows the patch to give you the +contributor the credit for your patch, and gives the Salt community an archive +of the patch and a place for discussion. +.SS Setting a Github pull request +.sp +This is probably the preferred method for contributions, simply create a Github +fork, commit your changes to the fork, and then open up a pull request. +.SS Contributions Welcome! +.sp +The goal here it to make contributions clear, make sure there is a trail for +where the code has come from, but most importantly, to give credit where credit +is due! +.SH INTRODUCTION TO EXTENDING SALT +.sp +Salt is made to be used, and made to be extended. The primary goal of Salt is +to provide a foundation which can be used to solve problems. And the goal of +Salt is to not assume what those problems might be. +.sp +One of the greatest benefit of developing Salt has been the vast array of ways +in which people have wanted to use it, while the original intention was as a +communication layer for a cloud controller Salt has been extended to facilitate +so much more. +.SS Client API +.sp +The primary interface used to extend salt, is to simply use it. Salt executions +can be called via the Salt client api, making programming master side solutions +with Salt is easy. +.SS Adding Loadable Plugins +.sp +Salt is comprised of a core platform that loads many types of easy to write +plugins. The idea is to enable all of the breaking points in the salt processes +to have a point of pluggable interaction. This means that all of the main +features of Salt can be extended, modified or used. +.sp +The breaking points and helping interfaces span from convenience master side +executions to manipulating the flow of how data is handled by Salt. +.SS Minion Execution Modules +.sp +The minion execution modules or just \fBmodules\fP are the core to what salt is +and does. These modules are found in: +.sp +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/modules\fP +.sp +These modules are what is called by the salt command line and the salt client +api. Adding modules is done by simply adding additional python modules to the +modules directory and restarting the minion. +.SS Grains +.sp +Salt grains, or "grains of truth" are bits of static information that are +generated when the minion starts. This information is useful when determining +what package manager to default to, or where certain configuration files are +stored on the minion. +.sp +The Salt grains are the interface used for auto detection and dynamic assignment +of execution modules and types to specific salt minions. +.sp +The code used to generate the Salt grains can be found here: +.sp +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/grains\fP +.SS States +.sp +Salt supports state enforcement, this makes Salt a high speed and very efficient +solution for system configuration management. +.sp +States can be easily added to Salt by dropping a new state module in: +.sp +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/states\fP +.SS Renderers +.sp +Salt states are controlled by simple data structures, these structures can be +abstracted in a number of ways. While the default is to be in a yaml file +wrapped in a jinja template, any abstraction can be used. This means that any +format that can be dreamed is possible, so long as a renderer is written for +it. +.sp +The existing renderers can be found here: +.sp +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/renderers\fP +.SS Returners +.sp +The salt commands all produce a return value, that return value is sent to the +salt master by default, but it can be sent anywhere. The returner interface +makes it programmatically possible for the information to be sent to anything +from an SQL or NOSQL database, to a custom application made to use Salt. +.sp +The existing returners can be found here: +.sp +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/returners\fP +.SS Runners +.sp +Sometimes a certain application can be made to execute and run from the +existing salt command line. This is where the salt runners come into play. +The Salt Runners what is called by the salt\-run command and are meant to +act as a generic interface for encapsulating master side executions. +.sp +Existing Salt runners are located here: +.sp +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/runners\fP +.SH MODULES +.sp +Salt modules are the functions called by the \fBsalt\fP command. +.IP "See also" +.sp +\fIFull list of builtin modules\fP +.sp +Salt ships with many modules that cover a wide variety of tasks. +.RE +.SS Easy Modules to write +.sp +Salt modules are amazingly simple to write, just write a regular Python module +or a regular Cython module and place it in the \fBsalt/modules\fP directory. You +can also place them in a directory called \fB_modules/\fP in your state directory. +.sp +Since Salt modules are just Python/Cython modules there are no restraints as to +what you can put inside of a salt module, and if a Salt module has errors and +cannot import the Salt minion will continue to load without issue, the module +with errors will simply be omitted. +.sp +If adding a Cython module the file must be named \fB<modulename>.pyx\fP so that +the loader knows that the module needs to be imported as a Cython module. The +compilation of the Cython module is automatic and happens when the minion +starts, so only the \fB*.pyx\fP file is required. +.SS Cross Calling Modules +.sp +All of the salt modules are available to each other, and can be "cross called". +This means that when creating a module functions in modules which already exist +can be called. +.sp +The variable \fB__salt__\fP is packed into the modules after they are loaded into +the salt minion. This variable is a python dictionary of all of the salt +functions, laid out in the same way that they are made available to the salt +command. +.sp +Salt modules can be cross called by accessing the value in the \fB__salt__\fP +dict: +.sp +.nf +.ft C +def foo(bar): + return __salt__[\(aqcmd.run\(aq](bar) +.ft P +.fi +.sp +This code will call the Salt cmd module\(aqs run function and pass the argument +\fBbar\fP. +.SS Preloaded Modules Data +.sp +When interacting with modules often it is nice to be able to read information +dynamically about the minion, or load in configuration parameters for a module. +Salt allows for different types of data to be loaded into the modules by the +minion, as of this writing Salt loads information gathered from the Salt Grains +system and from the minion configuration file. +.SS Grains Data +.sp +The Salt minion detects information about the system when started. This allows +for modules to be written dynamically with respect to the underlying hardware +and OS. This information is referred to as Salt Grains, or "grains of salt". +The Grains system was introduced to replace Facter, since relying on a Ruby +application from a Python application was both slow and inefficient. Grains +support replaces Facter in all releases after 0.8 +.sp +The values detected by the Salt Grains on the minion are available in a dict by +the name of \fB__grains__\fP and can be accessed from within callable objects in +the Python modules. +.sp +To see the contents of the grains dict for a given system in your deployment +run the \fBgrains.items()\fP function: +.sp +.nf +.ft C +salt \(aqhostname\(aq grains.items +.ft P +.fi +.sp +To use the \fB__grains__\fP dict simply call it as a Python dict from within your +code, an excellent example is available in the Grains module: +\fBsalt.modules.grains\fP. +.SS Module Configuration +.sp +Since parameters for configuring a module may be desired, Salt allows for +configuration information stored in the main minion config file to be passed to +the modules. +.sp +Since the minion configuration file is a yaml document, arbitrary configuration +data can be passed in the minion config that is read by the modules. It is +\fBstrongly\fP recommended that the values passed in the configuration file match +the module. This means that a value intended for the \fBtest\fP module should be +named \fBtest.<value>\fP. +.sp +Configuration also requires that default configuration parameters need to be +loaded as well. This can be done simply by adding the \fB__opts__\fP dict to the +top level of the module. +.sp +The test module contains usage of the module configuration, and the default +configuration file for the minion contains the information and format used to +pass data to the modules. \fBsalt.modules.test\fP, \fBconf/minion\fP. +.SS Printout Configuration +.sp +Since module functions can return different data, and the way the data is +printed can greatly change the presentation, Salt has a printout +configuration. +.sp +When writing a module the \fB__outputter__\fP dict can be declared in the module. +The \fB__outputter__\fP dict contains a mapping of function name to Salt +Outputter. +.sp +.nf +.ft C +__outputter__ = { + \(aqrun\(aq: \(aqtxt\(aq + } +.ft P +.fi +.sp +This will ensure that the text outputter is used. +.SS Virtual Modules +.sp +Sometimes a module should be presented in a generic way. A good example of this +can be found in the package manager modules. The package manager changes from +one operating system to another, but the salt module that interfaces with the +package manager can be presented in a generic way. +.sp +The salt modules for package managers all contain a \fB__virtual__\fP function +which is called to define what systems the module should be loaded on. +.sp +The \fB__virtual__\fP function is used to return either a string or False. If +False is returned then the module is not loaded, if a string is returned then +the module is loaded with the name of the string. +.sp +This means that the package manager modules can be presented as the pkg module +regardless of what the actual module is named. +.sp +The package manager modules are the best example of using the \fB__virtual__\fP +function: +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/modules/pacman.py\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/modules/yumpkg.py\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/modules/apt.py\fP +.SS Documentation +.sp +Salt modules are self documenting, the \fBsys.doc()\fP function will return the +documentation for all available Facter modules: +.sp +.nf +.ft C +salt \(aq*\(aq sys.doc +.ft P +.fi +.sp +This function simple prints out the docstrings found in the modules, when +writing salt modules, please follow the formating conventions for docstrings as +they appear in the other modules. +.SS Adding Documentation to Salt Modules +.sp +Since life is much better with documentation, it is strongly suggested that +all Salt modules have documentation added. Any Salt modules submitted for +inclusion in the main distribution of Salt will be required to have +documentation. +.sp +Documenting Salt modules is easy! Just add a python docstring to the function. +.sp +.nf +.ft C +def spam(eggs): + \(aq\(aq\(aq + A function to make some spam with eggs! + + CLI Example: + salt \(aq*\(aq test.spam eggs + \(aq\(aq\(aq + return eggs +.ft P +.fi +.sp +Now when the sys.doc call is executed the docstring will be cleanly returned +to the calling terminal. +.SS How Functions are Read +.sp +In Salt Python callable objects contained within a module are made available to +the Salt minion for use. The only exception to this rule is a callable object +with a name starting with an underscore \fB_\fP. +.SS Objects Loaded Into the Salt Minion +.sp +.nf +.ft C +def foo(bar): + return bar + +class baz: + def __init__(self, quo): + return quo +.ft P +.fi +.SS Objects NOT Loaded into the Salt Minion +.sp +.nf +.ft C +def _foobar(baz): # Preceded with an _ + return baz + +cheese = {} # Not a callable python object +.ft P +.fi +.SS Examples of Salt Modules +.sp +The existing Salt modules should be fairly easy to read and understand, the +goal of the main distribution\(aqs Salt modules is not only to build a set of +functions for salt, but to stand as examples for building out more Salt +modules. +.sp +The existing modules can be found here: +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/modules\fP +.sp +The most simple module is the test module, it contains the simplest salt +function, test.ping: +.sp +.nf +.ft C +def ping(): + \(aq\(aq\(aq + Just used to make sure the minion is up and responding + Return True + + CLI Example: + salt \(aq*\(aq test.ping + \(aq\(aq\(aq + return True +.ft P +.fi +.SH FULL LIST OF BUILTIN MODULES +.IP "Virtual modules" +.SS salt.modules.pkg +.sp +\fBpkg\fP is a virtual module that is fulfilled by one of the following modules: +.INDENT 0.0 +.IP \(bu 2 +\fBsalt.modules.apt\fP +.IP \(bu 2 +\fBsalt.modules.ebuild\fP +.IP \(bu 2 +\fBsalt.modules.freebsdpkg\fP +.IP \(bu 2 +\fBsalt.modules.pacman\fP +.IP \(bu 2 +\fBsalt.modules.yumpkg\fP +.IP \(bu 2 +\fBsalt.modules.yumpkg5\fP +.IP \(bu 2 +\fBsalt.modules.zypper\fP +.UNINDENT +.SS salt.modules.sys +.sp +The regular salt modules execute in a separate context from the salt minion +and manipulating the actual salt modules needs to happen in a higher level +context within the minion process. This is where the sys pseudo module is +used. +.sp +The sys pseudo module comes with a few functions that return data about the +available functions on the minion or allows for the minion modules to be +refreshed. These functions are as follows: +.INDENT 0.0 +.TP +.B salt.modules.sys.doc([module[, module.function]]) +Display the inline documentation for all available modules, or for the +specified module or function. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.sys.reload_modules() +Instruct the minion to reload all available modules in memory. This +function can be called if the modules need to be re\-evaluated for +availability or new modules have been made available to the minion. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.sys.list_modules() +List all available (loaded) modules. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.sys.list_functions() +List all known functions that are in available (loaded) modules. +.UNINDENT +.RE +.TS +center; +|l|l|. +_ +T{ +\fBapache\fP +T} T{ +Support for Apache +T} +_ +T{ +\fBapt\fP +T} T{ +Support for APT (Advanced Packaging Tool) +T} +_ +T{ +\fBarchive\fP +T} T{ +A module to wrap archive calls +T} +_ +T{ +\fBbutterkvm\fP +T} T{ +Specialized routines used by the butter cloud component +T} +_ +T{ +\fBcluster\fP +T} T{ +The cluster module is used to distribute and activate salt HA cluster +T} +_ +T{ +\fBcmdmod\fP +T} T{ +A module for shelling out +T} +_ +T{ +\fBcp\fP +T} T{ +Minion side functions for salt\-cp +T} +_ +T{ +\fBcron\fP +T} T{ +Work with cron +T} +_ +T{ +\fBdata\fP +T} T{ +Manage a local persistent data structure that can hold any arbitrairy data +T} +_ +T{ +\fBdebconfmod\fP +T} T{ +Support for Debconf +T} +_ +T{ +\fBdisk\fP +T} T{ +Module for gathering disk information +T} +_ +T{ +\fBebuild\fP +T} T{ +Support for Portage +T} +_ +T{ +\fBfile\fP +T} T{ +Manage information about files on the minion, set/read user, group, and mode +T} +_ +T{ +\fBfreebsdkmod\fP +T} T{ +Module to manage FreeBSD kernel modules +T} +_ +T{ +\fBfreebsdpkg\fP +T} T{ +Package support for FreeBSD +T} +_ +T{ +\fBfreebsdservice\fP +T} T{ +The service module for FreeBSD +T} +_ +T{ +\fBgem\fP +T} T{ +Manage ruby gems. +T} +_ +T{ +\fBgentoo_service\fP +T} T{ +Top level package command wrapper, used to translate the os detected by the +T} +_ +T{ +\fBgit\fP +T} T{ +Support for the Git SCM +T} +_ +T{ +\fBgrains\fP +T} T{ +Control aspects of the grains data +T} +_ +T{ +\fBgroupadd\fP +T} T{ +Manage groups on Linux +T} +_ +T{ +\fBhg\fP +T} T{ +Support for the Mercurial SCM +T} +_ +T{ +\fBhosts\fP +T} T{ +Manage the information in the hosts file +T} +_ +T{ +\fBkmod\fP +T} T{ +Module to manage Linux kernel modules +T} +_ +T{ +\fBkvm_hyper\fP +T} T{ +Provide the hyper module for kvm hypervisors. +T} +_ +T{ +\fBlinux_sysctl\fP +T} T{ +Module for viewing and modifying sysctl parameters +T} +_ +T{ +\fBmdadm\fP +T} T{ +Salt module to manage RAID arrays with mdadm +T} +_ +T{ +\fBmoosefs\fP +T} T{ +Module for gathering and managing information about MooseFS +T} +_ +T{ +\fBmount\fP +T} T{ +Salt module to manage unix mounts and the fstab file +T} +_ +T{ +\fBmysql\fP +T} T{ +Module to provide MySQL compatibility to salt. +T} +_ +T{ +\fBnetwork\fP +T} T{ +Module for gathering and managing network information +T} +_ +T{ +\fBnginx\fP +T} T{ +Support for nginx +T} +_ +T{ +\fBpacman\fP +T} T{ +A module to wrap pacman calls, since Arch is the best +T} +_ +T{ +\fBpillar\fP +T} T{ +Extract the pillar data for this minion +T} +_ +T{ +\fBpip\fP +T} T{ +Install Python packages with pip to either the system or a virtualenv +T} +_ +T{ +\fBps\fP +T} T{ +A salt interface to psutil, a system and process library. +T} +_ +T{ +\fBpublish\fP +T} T{ +Publish a command from a minion to a target +T} +_ +T{ +\fBpuppet\fP +T} T{ +Execute puppet routines +T} +_ +T{ +\fBpw_group\fP +T} T{ +Manage groups on Linux +T} +_ +T{ +\fBpw_user\fP +T} T{ +Manage users with the useradd command +T} +_ +T{ +\fBrh_service\fP +T} T{ +Service support for classic Red Hat type systems. This interface uses the +T} +_ +T{ +\fBrvm\fP +T} T{ +Manage ruby installations and gemsets with RVM, the Ruby Version Manager. +T} +_ +T{ +\fBsaltutil\fP +T} T{ +The Saltutil module is used to manage the state of the salt minion itself. It is +T} +_ +T{ +\fBselinux\fP +T} T{ +Execute calls on selinux +T} +_ T{ \fBservice\fP T} T{ @@ -1477,12 +3657,6 @@ Module for returning various status data about a minion. T} _ T{ -\fBsys\fP -T} T{ -This module provides access to some objects used or maintained by the interpreter and to functions that interact strongly with the interpreter. -T} -_ -T{ \fBsystemd\fP T} T{ Provide the service module for systemd @@ -1501,6 +3675,12 @@ Support for Tomcat T} _ T{ +\fBupstart\fP +T} T{ +Module for the management of upstart systems. +T} +_ +T{ \fBuseradd\fP T} T{ Manage users with the useradd command @@ -1525,12 +3705,30 @@ Module for gathering disk information on Windows T} _ T{ +\fBwin_file\fP +T} T{ +Manage information about files on the minion, set/read user, group +T} +_ +T{ +\fBwin_network\fP +T} T{ +Module for gathering and managing network information +T} +_ +T{ \fBwin_service\fP T} T{ Windows Service module. T} _ T{ +\fBwin_shadow\fP +T} T{ +Manage the shadow file +T} +_ +T{ \fBwin_useradd\fP T} T{ Manage Windows users with the net user command @@ -1548,111 +3746,1710 @@ T} T{ Support for YUM T} _ +T{ +\fBzypper\fP +T} T{ +Package support for openSUSE via the zypper package manager +T} +_ .TE .SS salt.modules.apache .sp -Support for Apache +Support for Apache +.INDENT 0.0 +.TP +.B salt.modules.apache.directives() +Return list of directives together with expected arguments +and places where the directive is valid (\fBapachectl \-L\fP) +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq apache.directives +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apache.fullversion() +Return server version from apachectl \-V +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq apache.fullversion +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apache.modules() +Return list of static and shared modules from apachectl \-M +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq apache.modules +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apache.servermods() +Return list of modules compiled into the server (apachectl \-l) +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq apache.servermods +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apache.signal(signal=None) +Signals httpd to start, restart, or stop. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq apache.signal restart +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apache.version() +Return server version from apachectl \-v +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq apache.version +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apache.vhosts() +Show the settings as parsed from the config file (currently +only shows the virtualhost settings). (\fBapachectl \-S\fP) +Because each additional virtual host adds to the execution +time, this command may require a long timeout be specified. +.sp +CLI Example: +.sp +.nf +.ft C +salt \-t 10 \(aq*\(aq apache.vhosts +.ft P +.fi +.UNINDENT +.SS salt.modules.apt +.sp +Support for APT (Advanced Packaging Tool) +.INDENT 0.0 +.TP +.B salt.modules.apt.available_version(name) +The available version of the package in the repository +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.available_version <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apt.install(pkg, refresh=False, repo=\(aq\(aq, skip_verify=False, debconf=None, **kwargs) +Install the passed package +.INDENT 7.0 +.TP +.B pkg +The name of the package to be installed +.TP +.B refresh +False +Update apt before continuing +.TP +.B repo +(default) +Specify a package repository to install from +(e.g., \fBapt\-get \-t unstable install somepackage\fP) +.TP +.B skip_verify +False +Skip the GPG verification check (e.g., \fB\-\-allow\-unauthenticated\fP) +.TP +.B debconf +None +Provide the path to a debconf answers file, processed before +installation. +.UNINDENT +.sp +Return a dict containing the new package names and versions: +.sp +.nf +.ft C +{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, + \(aqnew\(aq: \(aq<new\-version>\(aq]} +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.install <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apt.list_pkgs(regex_string=\(aq\(aq) +List the packages currently installed in a dict: +.sp +.nf +.ft C +{\(aq<package_name>\(aq: \(aq<version>\(aq} +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.list_pkgs +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apt.purge(pkg) +Remove a package via \fBapt\-get purge\fP along with all configuration +files and unused dependencies. +.sp +Returns a list containing the names of the removed packages +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.purge <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apt.refresh_db() +Updates the APT database to latest packages based upon repositories +.sp +Returns a dict: +.sp +.nf +.ft C +{\(aq<database name>\(aq: Bool} +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.refresh_db +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apt.remove(pkg) +Remove a single package via \fBapt\-get remove\fP +.sp +Returns a list containing the names of the removed packages. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.remove <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apt.upgrade(refresh=True) +Upgrades all packages via \fBapt\-get dist\-upgrade\fP +.sp +Returns a list of dicts containing the package names, and the new and old +versions: +.sp +.nf +.ft C +[ + {\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, + \(aqnew\(aq: \(aq<new\-version>\(aq] + }\(aq, + ... +] +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.upgrade +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apt.upgrade_available(name) +Check whether or not an upgrade is available for a given package +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.upgrade_available <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.apt.version(name) +Returns a string representing the package version or an empty string if not +installed +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.version <package name> +.ft P +.fi +.UNINDENT +.SS salt.modules.archive +.sp +A module to wrap archive calls +.INDENT 0.0 +.TP +.B salt.modules.archive.gunzip(gzipfile) +Uses the gunzip command to unpack gzip files +.sp +CLI Example to create \fB/tmp/sourcefile.txt\fP: +.sp +.nf +.ft C +salt \(aq*\(aq archive.gunzip /tmp/sourcefile.txt.gz +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.archive.gzip(sourcefile) +Uses the gzip command to create gzip files +.sp +CLI Example to create \fB/tmp/sourcefile.txt.gz\fP: +.sp +.nf +.ft C +salt \(aq*\(aq archive.gzip /tmp/sourcefile.txt +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.archive.rar(rarfile, *sources) +Uses the rar command to create rar files +Uses rar for Linux from \fI\%http://www.rarlab.com/\fP +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq archive.rar /tmp/rarfile.rar /tmp/sourcefile1 /tmp/sourcefile2 +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.archive.tar(options, tarfile, *sources) +Uses the tar command to pack, unpack, etc tar files +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq archive.tar cjvf /tmp/tarfile.tar.bz2 /tmp/file1 /tmp/file2 +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.archive.unrar(rarfile, dest, *xfiles) +Uses the unrar command to unpack rar files +Uses rar for Linux from \fI\%http://www.rarlab.com/\fP +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq archive.unrar /tmp/rarfile.rar /home/strongbad/ file1 file2 +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.archive.unzip(zipfile, dest, *xfiles) +Uses the unzip command to unpack zip files +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq archive.unzip /tmp/zipfile.zip /home/strongbad/ file1 file2 +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.archive.zip(zipfile, *sources) +Uses the zip command to create zip files +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq archive.zip /tmp/zipfile.zip /tmp/sourcefile1 /tmp/sourcefile2 +.ft P +.fi +.UNINDENT +.SS salt.modules.butterkvm +.sp +Specialized routines used by the butter cloud component +.INDENT 0.0 +.TP +.B salt.modules.butterkvm.create(instance, vda, image, pin) +Create a virtual machine, this is part of the butter vm system and assumes +that the files prepared by butter are available via shared storage. +AKA \- don\(aqt call this from the command line! +.INDENT 7.0 +.TP +.B instance +string +The path to the instance directory for the given vm on shared storage +.TP +.B vda +The location where the virtual machine image needs to be placed +.TP +.B image +The image to move into place +.TP +.B pin +A "pin" data structure defining the myriad of possible vdb\-vbz disk +images to generate +.UNINDENT +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq butterkvm.create <instance dir> <root image location>\e + <Destination> <pin data> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.butterkvm.full_butter_data(local_path) +Return the full virt info, but add butter data! +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq buttervm.full_butter_data <image_path> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.butterkvm.libvirt_creds() +Returns the user and group that the disk images should be owned by +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq butterkvm.libvirt_creds +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.butterkvm.local_images(local_path) +return the virtual machine names for all of the images located in the +butter cloud\(aqs local_path in a list: +.sp +.nf +.ft C +[\(aqvm1.boo.com\(aq, \(aqvm2.foo.com\(aq] +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq buttervm.local_images <image_path> +.ft P +.fi +.UNINDENT +.SS salt.modules.cluster +.sp +The cluster module is used to distribute and activate salt HA cluster +components +.INDENT 0.0 +.TP +.B salt.modules.cluster.distrib(minions, master_conf, master_pem, conf_file) +Set up this minion as a failover master \- only intended for use by the +cluster interface +.UNINDENT +.SS salt.modules.cmd +.sp +A module for shelling out +.sp +Keep in mind that this module is insecure, in that it can give whomever has +access to the master root execution access to all salt minions +.INDENT 0.0 +.TP +.B salt.modules.cmdmod.exec_code(lang, code, cwd=None) +Pass in two strings, the first naming the executable language, aka \- +python2, python3, ruby, perl, lua, etc. the second string containing +the code you wish to execute. The stdout and stderr will be returned +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cmd.exec_code ruby \(aqputs "cheese"\(aq +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cmdmod.has_exec(cmd) +Returns true if the executable is available on the minion, false otherwise +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cmd.has_exec cat +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cmdmod.retcode(cmd, cwd=None, runas=None, shell=\(aq/bin/sh\(aq, env=()) +Execute a shell command and return the command\(aqs return code. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cmd.retcode "file /bin/bash" +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cmdmod.run(cmd, cwd=None, runas=None, shell=\(aq/bin/sh\(aq, env=()) +Execute the passed command and return the output as a string +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cmd.run "ls \-l | awk \(aq/foo/{print $2}\(aq" +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cmdmod.run_all(cmd, cwd=None, runas=None, shell=\(aq/bin/sh\(aq, env=()) +Execute the passed command and return a dict of return data +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cmd.run_all "ls \-l | awk \(aq/foo/{print $2}\(aq" +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cmdmod.run_stderr(cmd, cwd=None, runas=None, shell=\(aq/bin/sh\(aq, env=()) +Execute a command and only return the standard error +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cmd.run_stderr "ls \-l | awk \(aq/foo/{print $2}\(aq" +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cmdmod.run_stdout(cmd, cwd=None, runas=None, shell=\(aq/bin/sh\(aq, env=()) +Execute a command, and only return the standard out +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cmd.run_stdout "ls \-l | awk \(aq/foo/{print $2}\(aq" +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cmdmod.which(cmd) +Returns the path of an executable available on the minion, None otherwise +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cmd.which cat +.ft P +.fi +.UNINDENT +.SS salt.modules.cp +.sp +Minion side functions for salt\-cp +.INDENT 0.0 +.TP +.B salt.modules.cp.cache_dir(path, env=\(aqbase\(aq) +Download and cache everything under a directory from the master +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cp.cache_dir salt://path/to/dir +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.cache_file(path, env=\(aqbase\(aq) +Used to cache a single file in the local salt\-master file cache. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cp.cache_file salt://path/to/file +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.cache_files(paths, env=\(aqbase\(aq) +Used to gather many files from the master, the gathered files will be +saved in the minion cachedir reflective to the paths retrieved from the +master. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cp.cache_files salt://pathto/file1,salt://pathto/file1 +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.cache_local_file(path) +Cache a local file on the minion in the localfiles cache +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cp.cache_local_file /etc/hosts +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.cache_master(env=\(aqbase\(aq) +Retrieve all of the files on the master and cache them locally +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cp.cache_master +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.get_dir(path, dest, env=\(aqbase\(aq) +Used to recursively copy a directory from the salt master +.sp +CLI Example: +.INDENT 7.0 +.INDENT 3.5 +salt \(aq*\(aq cp.get_dir salt://path/to/dir/ /minion/dest +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.get_file(path, dest, env=\(aqbase\(aq) +Used to get a single file from the salt master +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cp.get_file salt://path/to/file /minion/dest +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.get_url(path, dest, env=\(aqbase\(aq) +Used to get a single file from a URL. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cp.get_url salt://my/file /tmp/mine +salt \(aq*\(aq cp.get_url http://www.slashdot.org /tmp/index.html +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.hash_file(path, env=\(aqbase\(aq) +Return the hash of a file, to get the hash of a file on the +salt master file server prepend the path with salt://<file on server> +otherwise, prepend the file with / for a local file. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cp.hash_file salt://path/to/file +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.is_cached(path, env=\(aqbase\(aq) +Return a boolean if the given path on the master has been cached on the +minion +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cp.is_cached salt://path/to/file +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.list_master(env=\(aqbase\(aq) +List all of the files stored on the master +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cp.list_master +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.list_minion(env=\(aqbase\(aq) +List all of the files cached on the minion +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cp.list_minion +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cp.recv(files, dest) +Used with salt\-cp, pass the files dict, and the destination. +.sp +This function receives small fast copy files from the master via salt\-cp +.UNINDENT +.SS salt.modules.cron +.sp +Work with cron +.INDENT 0.0 +.TP +.B salt.modules.cron.list_tab(user) +Return the contents of the specified user\(aqs crontab +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cron.list_tab root +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cron.ls(user) +Return the contents of the specified user\(aqs crontab +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cron.list_tab root +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cron.raw_cron(user) +Return the contents of the user\(aqs crontab +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cron.rm(user, minute, hour, dom, month, dow, cmd) +Remove a cron job up for a specified user. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cron.rm_job(user, minute, hour, dom, month, dow, cmd) +Remove a cron job up for a specified user. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cron.set_job(user, minute, hour, dom, month, dow, cmd) +Sets a cron job up for a specified user. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.cron.set_special(user, special, cmd) +Set up a special command in the crontab. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq cron.set_special @hourly \(aqecho foobar\(aq +.ft P +.fi +.UNINDENT +.SS salt.modules.data +.sp +Manage a local persistent data structure that can hold any arbitrairy data +specific to the minion +.INDENT 0.0 +.TP +.B salt.modules.data.dump(new_data) +Replace the entire datastore with a passed data structure +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq data.dump \(aq{\(aqeggs\(aq: \(aqspam\(aq}\(aq +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.data.getval(key) +Get a value from the minion datastore +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq data.getval <key> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.data.getvals(keys) +Get values from the minion datastore +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq data.getvals <key> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.data.load() +Return all of the data in the minion datastore +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq data.load +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.data.update(key, value) +Update a key with a value in the minion datastore +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq data.update <key> <value> +.ft P +.fi +.UNINDENT +.SS salt.modules.debconfmod +.sp +Support for Debconf +.INDENT 0.0 +.TP +.B salt.modules.debconfmod.get_selections(fetchempty=True) +Answers to debconf questions for all packages in the following format: +.sp +.nf +.ft C +{\(aqpackage\(aq: [[\(aqquestion\(aq, \(aqtype\(aq, \(aqvalue\(aq], ...]} +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq debconf.get_selections +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.debconfmod.set(package, question, type, value, *extra) +Set answers to debconf questions for a package. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq debconf.set <package> <question> <type> <value> [<value> ...] +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.debconfmod.set_file(path) +Set answers to debconf questions from a file. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq debconf.set_file salt://pathto/pkg.selections +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.debconfmod.show(name) +Answers to debconf questions for a package in the following format: +.sp +.nf +.ft C +[[\(aqquestion\(aq, \(aqtype\(aq, \(aqvalue\(aq], ...] +.ft P +.fi +.sp +If debconf doesn\(aqt know about a package, we return None. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq debconf.show <package name> +.ft P +.fi +.UNINDENT +.SS salt.modules.disk +.sp +Module for gathering disk information +.INDENT 0.0 +.TP +.B salt.modules.disk.inodeusage() +Return inode usage information for volumes mounted on this minion +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq disk.inodeusage +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.disk.usage() +Return usage information for volumes mounted on this minion +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq disk.usage +.ft P +.fi +.UNINDENT +.SS salt.modules.ebuild +.sp +Support for Portage +.INDENT 0.0 +.TP +.B salt.modules.ebuild.available_version(name) +The available version of the package in the repository +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.available_version <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.ebuild.install(pkg, refresh=False, **kwargs) +Install the passed package +.sp +Return a dict containing the new package names and versions: +.sp +.nf +.ft C +{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, + \(aqnew\(aq: \(aq<new\-version>\(aq]} +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.install <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.ebuild.list_pkgs() +List the packages currently installed in a dict: +.sp +.nf +.ft C +{\(aq<package_name>\(aq: \(aq<version>\(aq} +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.list_pkgs +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.ebuild.purge(pkg) +Portage does not have a purge, this function calls remove +.sp +Return a list containing the removed packages: +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.purge <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.ebuild.refresh_db() +Updates the portage tree (emerge \-\-sync) +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.refresh_db +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.ebuild.remove(pkg) +Remove a single package via emerge \-\-unmerge +.sp +Return a list containing the names of the removed packages: +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.remove <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.ebuild.update(pkg, refresh=False) +Updates the passed package (emerge \-\-update package) +.sp +Return a dict containing the new package names and versions: +.sp +.nf +.ft C +{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, + \(aqnew\(aq: \(aq<new\-version>\(aq]} +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.update <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.ebuild.upgrade(refresh=False) +Run a full system upgrade (emerge \-\-update world) +.sp +Return a dict containing the new package names and versions: +.sp +.nf +.ft C +{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, + \(aqnew\(aq: \(aq<new\-version>\(aq]} +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.upgrade +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.ebuild.version(name) +Returns a version if the package is installed, else returns an empty string +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.version <package name> +.ft P +.fi +.UNINDENT +.SS salt.modules.file +.sp +Manage information about files on the minion, set/read user, group, and mode +data +.INDENT 0.0 +.TP +.B salt.modules.file.append(path, *args) +Append text to the end of a file +.sp +Usage: +.sp +.nf +.ft C +salt \(aq*\(aq file.append /etc/motd \e + "With all thine offerings thou shalt offer salt."\e + "Salt is what makes things taste bad when it isn\(aqt in them." +.ft P +.fi +.sp +New in version 0.9.5. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.chgrp(path, group) +Change the group of a file +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq file.chgrp /etc/passwd root +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.chown(path, user, group) +Chown a file, pass the file the desired user and group +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq file.chown /etc/passwd root root +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.comment(path, regex, char=\(aq#\(aq, backup=\(aq.bak\(aq) +Comment out specified lines in a file +.INDENT 7.0 +.TP +.B path +The full path to the file to be edited +.TP +.B regex +A regular expression used to find the lines that are to be commented; +this pattern will be wrapped in parenthesis and will move any +preceding/trailing \fB^\fP or \fB$\fP characters outside the parenthesis +(e.g., the pattern \fB^foo$\fP will be rewritten as \fB^(foo)$\fP) +.TP +.B char +\fB#\fP +The character to be inserted at the beginning of a line in order to +comment it out +.TP +.B backup +\fB.bak\fP +The file will be backed up before edit with this file extension +.IP Warning +This backup will be overwritten each time \fBsed\fP / \fBcomment\fP / +\fBuncomment\fP is called. Meaning the backup will only be useful +after the first invocation. +.RE +.UNINDENT +.sp +Usage: +.sp +.nf +.ft C +salt \(aq*\(aq file.comment /etc/modules pcspkr +.ft P +.fi +.sp +New in version 0.9.5. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.contains(path, text, limit=\(aq\(aq, escape=False) +Return True if the file at \fBpath\fP contains \fBtext\fP +.sp +Usage: +.sp +.nf +.ft C +salt \(aq*\(aq file.contains /etc/crontab \(aqmymaintenance.sh\(aq +.ft P +.fi +.sp +New in version 0.9.5. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.find(path, *opts) +Approximate the Unix find(1) command and return a list of paths that +meet the specified critera. +.sp +The options include match criteria: +.sp +.nf +.ft C +name = path\-glob # case sensitive +iname = path\-glob # case insensitive +regex = path\-regex # case sensitive +iregex = path\-regex # case insensitive +type = file\-types # match any listed type +user = users # match any listed user +group = groups # match any listed group +size = [+\-]number[size\-unit] # default unit = byte +mtime = interval # modified since date +grep = regex # search file contents +.ft P +.fi +.sp +and/or actions: +.sp +.nf +.ft C +delete [= file\-types] # default type = \(aqf\(aq +exec = command [arg ...] # where {} is replaced by pathname +print [= print\-opts] +.ft P +.fi +.sp +The default action is \(aqprint=path\(aq. +.sp +file\-glob: +.sp +.nf +.ft C +* = match zero or more chars +? = match any char +[abc] = match a, b, or c +[!abc] or [^abc] = match anything except a, b, and c +[x\-y] = match chars x through y +[!x\-y] or [^x\-y] = match anything except chars x through y +{a,b,c} = match a or b or c +.ft P +.fi +.sp +path\-regex: a Python re (regular expression) pattern to match pathnames +.sp +file\-types: a string of one or more of the following: +.sp +.nf +.ft C +a: all file types +b: block device +c: character device +d: directory +p: FIFO (named pipe) +f: plain file +l: symlink +s: socket +.ft P +.fi +.sp +users: a space and/or comma separated list of user names and/or uids +.sp +groups: a space and/or comma separated list of group names and/or gids +.sp +size\-unit: +.sp +.nf +.ft C +b: bytes +k: kilobytes +m: megabytes +g: gigabytes +t: terabytes +.ft P +.fi +.sp +interval: +.sp +.nf +.ft C +[<num>w] [<num>[d]] [<num>h] [<num>m] [<num>s] + +where: + w: week + d: day + h: hour + m: minute + s: second +.ft P +.fi +.sp +print\-opts: a comma and/or space separated list of one or more of the +following: +.sp +.nf +.ft C +group: group name +md5: MD5 digest of file contents +mode: file permissions (as integer) +mtime: last modification time (as time_t) +name: file basename +path: file absolute path +size: file size in bytes +type: file type +user: user name +.ft P +.fi +.sp +CLI Examples: +.sp +.nf +.ft C +salt \(aq*\(aq file.find / type=f name=\e*.bak size=+10m +salt \(aq*\(aq file.find /var mtime=+30d size=+10m print=path,size,mtime +salt \(aq*\(aq file.find /var/log name=\e*.[0\-9] mtime=+30d size=+10m delete +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.get_gid(path) +Return the id of the group that owns a given file +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq file.get_gid /etc/passwd +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.get_group(path) +Return the group that owns a given file +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq file.get_group /etc/passwd +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.get_mode(path) +Return the mode of a file +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq file.get_mode /etc/passwd +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.get_sum(path, form=\(aqmd5\(aq) +Return the sum for the given file, default is md5, sha1, sha224, sha256, +sha384, sha512 are supported +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq file.get_sum /etc/passwd sha512 +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.get_uid(path) +Return the id of the user that owns a given file +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq file.get_uid /etc/passwd +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.get_user(path) +Return the user that owns a given file +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq file.get_user /etc/passwd +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.gid_to_group(gid) +Convert the group id to the group name on this system +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq file.gid_to_group 0 +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.group_to_gid(group) +Convert the group to the gid on this system +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq file.group_to_gid root +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.sed(path, before, after, limit=\(aq\(aq, backup=\(aq.bak\(aq, options=\(aq\-r \-e\(aq, flags=\(aqg\(aq, escape_all=False) +Make a simple edit to a file +.sp +Equivalent to: +.sp +.nf +.ft C +sed <backup> <options> "/<limit>/ s/<before>/<after>/<flags> <file>" +.ft P +.fi +.INDENT 7.0 +.TP +.B path +The full path to the file to be edited +.TP +.B before +A pattern to find in order to replace with \fBafter\fP +.TP +.B after +Text that will replace \fBbefore\fP +.TP +.B limit +\fB\(aq\(aq\fP +An initial pattern to search for before searching for \fBbefore\fP +.TP +.B backup +\fB.bak\fP +The file will be backed up before edit with this file extension; +\fBWARNING:\fP each time \fBsed\fP/\fBcomment\fP/\fBuncomment\fP is called will +overwrite this backup +.TP +.B options +\fB\-r \-e\fP +Options to pass to sed +.TP +.B flags +\fBg\fP +Flags to modify the sed search; e.g., \fBi\fP for case\-insensitve pattern +matching +.UNINDENT +.sp +Forward slashes and single quotes will be escaped automatically in the +\fBbefore\fP and \fBafter\fP patterns. +.sp +Usage: +.sp +.nf +.ft C +salt \(aq*\(aq file.sed /etc/httpd/httpd.conf \(aqLogLevel warn\(aq \(aqLogLevel info\(aq +.ft P +.fi +.sp +New in version 0.9.5. +.UNINDENT .INDENT 0.0 .TP -.B salt.modules.apache.directives() -Return list of directives together with expected arguments -and places where the directive is valid (\fBapachectl \-L\fP) +.B salt.modules.file.set_mode(path, mode) +Set the mode of a file .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq apache.directives +salt \(aq*\(aq file.set_mode /etc/passwd 0644 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apache.fullversion() -Return server version from apachectl \-V +.B salt.modules.file.touch(name, atime=None, mtime=None) +Just like \(aqnix\(aqs "touch" command, create a file if it +doesn\(aqt exist or simply update the atime and mtime if +it already does. +.INDENT 7.0 +.TP +.B atime: +Access time in Unix epoch time +.TP +.B mtime: +Last modification in Unix epoch time +.TP +.B Usage:: +salt \(aq*\(aq file.touch /var/log/emptyfile +.UNINDENT +.sp +New in version 0.9.5. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.uid_to_user(uid) +Convert a uid to a user name .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq apache.fullversion +salt \(aq*\(aq file.uid_to_user 0 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apache.modules() -Return list of static and shared modules from apachectl \-M +.B salt.modules.file.uncomment(path, regex, char=\(aq#\(aq, backup=\(aq.bak\(aq) +Uncomment specified commented lines in a file +.INDENT 7.0 +.TP +.B path +The full path to the file to be edited +.TP +.B regex +A regular expression used to find the lines that are to be uncommented. +This regex should not include the comment character. A leading \fB^\fP +character will be stripped for convenience (for easily switching +between comment() and uncomment()). +.TP +.B char +\fB#\fP +The character to remove in order to uncomment a line; if a single +whitespace character follows the comment it will also be removed +.TP +.B backup +\fB.bak\fP +The file will be backed up before edit with this file extension; +\fBWARNING:\fP each time \fBsed\fP/\fBcomment\fP/\fBuncomment\fP is called will +overwrite this backup +.UNINDENT +.sp +Usage: +.sp +.nf +.ft C +salt \(aq*\(aq file.uncomment /etc/hosts.deny \(aqALL: PARANOID\(aq +.ft P +.fi +.sp +New in version 0.9.5. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.file.user_to_uid(user) +Convert user name to a uid .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq apache.modules +salt \(aq*\(aq file.user_to_uid root +.ft P +.fi +.UNINDENT +.SS salt.modules.freebsdkmod +.sp +Module to manage FreeBSD kernel modules +.INDENT 0.0 +.TP +.B salt.modules.freebsdkmod.available() +Return a list of all available kernel modules +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq kmod.available .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apache.servermods() -Return list of modules compiled into the server (apachectl \-l) +.B salt.modules.freebsdkmod.check_available(mod) +Check to see if the specified kernel module is available .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq apache.servermods +salt \(aq*\(aq kmod.check_available kvm .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apache.signal(signal=None) -Signals httpd to start, restart, or stop. +.B salt.modules.freebsdkmod.load(mod) +Load the specified kernel module .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq apache.signal restart +salt \(aq*\(aq kmod.load kvm .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apache.version() -Return server version from apachectl \-v +.B salt.modules.freebsdkmod.lsmod() +Return a dict containing information about currently loaded modules .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq apache.version +salt \(aq*\(aq kmod.lsmod .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apache.vhosts() -Show the settings as parsed from the config file (currently -only shows the virtualhost settings). (\fBapachectl \-S\fP) -Because each additional virtual host adds to the execution -time, this command may require a long timeout be specified. +.B salt.modules.freebsdkmod.remove(mod) +Remove the specified kernel module .sp CLI Example: .sp .nf .ft C -salt \-t 10 \(aq*\(aq apache.vhosts +salt \(aq*\(aq kmod.remove kvm .ft P .fi .UNINDENT -.SS salt.modules.apt +.SS salt.modules.freebsdpkg .sp -Support for APT (Advanced Packaging Tool) +Package support for FreeBSD .INDENT 0.0 .TP -.B salt.modules.apt.available_version(name) +.B salt.modules.freebsdpkg.available_version(name) The available version of the package in the repository .sp CLI Example: @@ -1665,33 +5462,15 @@ salt \(aq*\(aq pkg.available_version <package name> .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apt.install(pkg, refresh=False, repo=\(aq\(aq, skip_verify=False, **kwargs) +.B salt.modules.freebsdpkg.install(name, *args, **kwargs) Install the passed package -.INDENT 7.0 -.TP -.B pkg -The name of the package to be installed -.TP -.B refresh -False -Update apt before continuing -.TP -.B repo -(default) -Specify a package repository to install from -(e.g., \fBapt\-get \-t unstable install somepackage\fP) -.TP -.B skip_verify -False -Skip the GPG verification check (e.g., \fB\-\-allow\-unauthenticated\fP) -.UNINDENT .sp Return a dict containing the new package names and versions: .sp .nf .ft C {\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, - \(aqnew\(aq: \(aq<new\-version>\(aq]} + \(aqnew\(aq: \(aq<new\-version>\(aq]} .ft P .fi .sp @@ -1705,8 +5484,8 @@ salt \(aq*\(aq pkg.install <package name> .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apt.list_pkgs(regex_string=\(aq\(aq) -List the packages currently installed in a dict: +.B salt.modules.freebsdpkg.list_pkgs() +List the packages currently installed as a dict: .sp .nf .ft C @@ -1724,11 +5503,10 @@ salt \(aq*\(aq pkg.list_pkgs .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apt.purge(pkg) -Remove a package via \fBapt\-get purge\fP along with all configuration -files and unused dependencies. +.B salt.modules.freebsdpkg.purge(name) +Remove a single package with pkg_delete .sp -Returns a list containing the names of the removed packages +Returns a list containing the removed packages. .sp CLI Example: .sp @@ -1740,822 +5518,1336 @@ salt \(aq*\(aq pkg.purge <package name> .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apt.refresh_db() -Updates the APT database to latest packages based upon repositories +.B salt.modules.freebsdpkg.refresh_db() +Update the ports tree with portsnap. If the ports tree does not exist it +will be downloaded and set up. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.refresh_db +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.freebsdpkg.rehash() +Recomputes internal hash table for the PATH variable. +Use whenever a new command is created during the current +session. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.rehash +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.freebsdpkg.remove(name) +Remove a single package with pkg_delete +.sp +Returns a list containing the removed packages. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.remove <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.freebsdpkg.upgrade() +Run a full system upgrade, a \fBfreebsd\-update fetch install\fP +.sp +Return a dict containing the new package names and versions: +.sp +.nf +.ft C +{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, + \(aqnew\(aq: \(aq<new\-version>\(aq]} +.ft P +.fi .sp -Returns a dict: +CLI Example: .sp .nf .ft C -{\(aq<database name>\(aq: Bool} +salt \(aq*\(aq pkg.upgrade .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.freebsdpkg.version(name) +Returns a version if the package is installed, else returns an empty string .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.refresh_db +salt \(aq*\(aq pkg.version <package name> .ft P .fi .UNINDENT +.SS salt.modules.freebsdservice +.sp +The service module for FreeBSD .INDENT 0.0 .TP -.B salt.modules.apt.remove(pkg) -Remove a single package via \fBapt\-get remove\fP -.sp -Returns a list containing the names of the removed packages. +.B salt.modules.freebsdservice.get_all() +Return a list of all available services +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.freebsdservice.get_disabled() +Return what services are available but not enabled to start at boot +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.freebsdservice.get_enabled() +Return what services are set to run on boot +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.freebsdservice.restart(name) +Restart the named service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.remove <package name> +salt \(aq*\(aq service.restart <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apt.upgrade(refresh=True) -Upgrades all packages via \fBapt\-get dist\-upgrade\fP +.B salt.modules.freebsdservice.start(name) +Start the specified service .sp -Returns a list of dicts containing the package names, and the new and old -versions: +CLI Example: .sp .nf .ft C -[ - {\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, - \(aqnew\(aq: \(aq<new\-version>\(aq] - }\(aq, - ... -] +salt \(aq*\(aq service.start <service name> .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.freebsdservice.status(name, sig=None) +Return the status for a service, returns the PID or an empty string if the +service is running or not, pass a signature to use to find the service via +ps .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.upgrade +salt \(aq*\(aq service.status <service name> [service signature] .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.apt.version(name) -Returns a string representing the package version or an empty string if not -installed +.B salt.modules.freebsdservice.stop(name) +Stop the specified service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.version <package name> +salt \(aq*\(aq service.stop <service name> .ft P .fi .UNINDENT -.SS salt.modules.archive +.SS salt.modules.gem .sp -A module to wrap archive calls +Manage ruby gems. .INDENT 0.0 .TP -.B salt.modules.archive.gunzip(gzipfile) -Uses the gunzip command to unpack gzip files +.B salt.modules.gem.install(gems, ruby=None, runas=None) +Installs one or several gems. +.INDENT 7.0 +.TP +.B gems +The gems to install. +.TP +.B ruby +None +If RVM is installed, the ruby version and gemset to use. +.TP +.B runas +None +The user to run gem as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.gem.list(prefix=\(aq\(aq, ruby=None, runas=None) +List locally installed gems. +.INDENT 7.0 +.TP +.B prefix : +Only list gems when the name matches this prefix. +.TP +.B ruby +None +If RVM is installed, the ruby version and gemset to use. +.TP +.B runas +None +The user to run gem as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.gem.sources_add(source_uri, ruby=None, runas=None) +Add a gem source. +.INDENT 7.0 +.TP +.B source_uri +The source URI to add. +.TP +.B ruby +None +If RVM is installed, the ruby version and gemset to use. +.TP +.B runas +None +The user to run gem as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.gem.sources_list(ruby=None, runas=None) +List the configured gem sources. +.INDENT 7.0 +.TP +.B ruby +None +If RVM is installed, the ruby version and gemset to use. +.TP +.B runas +None +The user to run gem as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.gem.sources_remove(source_uri, ruby=None, runas=None) +Remove a gem source. +.INDENT 7.0 +.TP +.B source_uri +The source URI to remove. +.TP +.B ruby +None +If RVM is installed, the ruby version and gemset to use. +.TP +.B runas +None +The user to run gem as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.gem.uninstall(gems, ruby=None, runas=None) +Uninstall one or several gems. +.INDENT 7.0 +.TP +.B gems +The gems to uninstall. +.TP +.B ruby +None +If RVM is installed, the ruby version and gemset to use. +.TP +.B runas +None +The user to run gem as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.gem.update(gems, ruby=None, runas=None) +Update one or several gems. +.INDENT 7.0 +.TP +.B gems +The gems to update. +.TP +.B ruby +None +If RVM is installed, the ruby version and gemset to use. +.TP +.B runas +None +The user to run gem as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.gem.update_system(version=\(aq\(aq, ruby=None, runas=None) +Update rubygems. +.INDENT 7.0 +.TP +.B version +(newest) +The version of rubygems to install. +.TP +.B ruby +None +If RVM is installed, the ruby version and gemset to use. +.TP +.B runas +None +The user to run gem as. +.UNINDENT +.UNINDENT +.SS salt.modules.gentoo_service .sp -CLI Example to create \fB/tmp/sourcefile.txt\fP: +Top level package command wrapper, used to translate the os detected by the +grains to the correct service manager +.INDENT 0.0 +.TP +.B salt.modules.gentoo_service.disable(name) +Disable the named service to start at boot +.sp +CLI Example: .sp .nf .ft C -salt \(aq*\(aq archive.gunzip /tmp/sourcefile.txt.gz +salt \(aq*\(aq service.disable <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.archive.gzip(sourcefile) -Uses the gzip command to create gzip files +.B salt.modules.gentoo_service.disabled(name) +Return True if the named servioce is enabled, false otherwise .sp -CLI Example to create \fB/tmp/sourcefile.txt.gz\fP: +CLI Example: .sp .nf .ft C -salt \(aq*\(aq archive.gzip /tmp/sourcefile.txt +salt \(aq*\(aq service.enabled <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.archive.rar(rarfile, *sources) -Uses the rar command to create rar files -Uses rar for Linux from \fI\%http://www.rarlab.com/\fP +.B salt.modules.gentoo_service.enable(name) +Enable the named service to start at boot .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq archive.rar /tmp/rarfile.rar /tmp/sourcefile1 /tmp/sourcefile2 +salt \(aq*\(aq service.enable <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.archive.tar(options, tarfile, *sources) -Uses the tar command to pack, unpack, etc tar files +.B salt.modules.gentoo_service.enabled(name) +Return True if the named servioce is enabled, false otherwise .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq archive.tar cjvf /tmp/tarfile.tar.bz2 /tmp/file1 /tmp/file2 +salt \(aq*\(aq service.enabled <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.archive.unrar(rarfile, dest, *xfiles) -Uses the unrar command to unpack rar files -Uses rar for Linux from \fI\%http://www.rarlab.com/\fP +.B salt.modules.gentoo_service.get_all() +Return all available boot services .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq archive.unrar /tmp/rarfile.rar /home/strongbad/ file1 file2 +salt \(aq*\(aq service.get_enabled .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.archive.unzip(zipfile, dest, *xfiles) -Uses the unzip command to unpack zip files +.B salt.modules.gentoo_service.get_disabled() +Return a set of services that are installed but disabled .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq archive.unzip /tmp/zipfile.zip /home/strongbad/ file1 file2 +salt \(aq*\(aq service.get_enabled .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.archive.zip(zipfile, *sources) -Uses the zip command to create zip files +.B salt.modules.gentoo_service.get_enabled() +Return a list of service that are enabled on boot .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq archive.zip /tmp/zipfile.zip /tmp/sourcefile1 /tmp/sourcefile2 +salt \(aq*\(aq service.get_enabled .ft P .fi .UNINDENT -.SS salt.modules.butterkvm -.sp -Specialized routines used by the butter cloud component .INDENT 0.0 .TP -.B salt.modules.butterkvm.create(instance, vda, image, pin) -Create a virtual machine, this is part of the butter vm system and assumes -that the files prepared by butter are available via shared storage. -AKA \- don\(aqt call this from the command line! -.INDENT 7.0 -.TP -.B instance -string -The path to the instance directory for the given vm on shared storage -.TP -.B vda -The location where the virtual machine image needs to be placed -.TP -.B image -The image to move into place -.TP -.B pin -A "pin" data structure defining the myriad of possible vdb\-vbz disk -images to generate -.UNINDENT +.B salt.modules.gentoo_service.restart(name) +Restart the named service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq butterkvm.create <instance dir> <root image location>\e - <Destination> <pin data> +salt \(aq*\(aq service.restart <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.butterkvm.full_butter_data(local_path) -Return the full virt info, but add butter data! +.B salt.modules.gentoo_service.start(name) +Start the specified service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq buttervm.full_butter_data <image_path> +salt \(aq*\(aq service.start <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.butterkvm.libvirt_creds() -Returns the user and group that the disk images should be owned by +.B salt.modules.gentoo_service.status(name, sig=None) +Return the status for a service, returns the PID or an empty string if the +service is running or not, pass a signature to use to find the service via +ps .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq butterkvm.libvirt_creds +salt \(aq*\(aq service.status <service name> [service signature] .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.butterkvm.local_images(local_path) -return the virtual machine names for all of the images located in the -butter cloud\(aqs local_path in a list: +.B salt.modules.gentoo_service.stop(name) +Stop the specified service +.sp +CLI Example: .sp .nf .ft C -[\(aqvm1.boo.com\(aq, \(aqvm2.foo.com\(aq] +salt \(aq*\(aq service.stop <service name> .ft P .fi +.UNINDENT +.SS salt.modules.git +.sp +Support for the Git SCM +.INDENT 0.0 +.TP +.B salt.modules.git.archive(cwd, output, rev=\(aqHEAD\(aq, fmt=None, prefix=None, user=None) +Export a tarball from the repository +.INDENT 7.0 +.TP +.B cwd +The path to the Git repository +.TP +.B output +The path to the archive tarball +.TP +.B rev: HEAD +The revision to create an archive from +.TP +.B fmt: None +Format of the resulting archive, zip and tar are commonly used +.TP +.B prefix +None +Prepend <prefix>/ to every filename in the archive +.TP +.B user +None +Run git as a user other than what the minion runs as +.UNINDENT +.sp +If \fBprefix\fP is not specified it defaults to the basename of the repo +directory. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq buttervm.local_images <image_path> +salt \(aq*\(aq git.archive /path/to/repo /path/to/archive.tar.gz .ft P .fi .UNINDENT -.SS salt.modules.cluster -.sp -The cluster module is used to distribute and activate salt HA cluster -components .INDENT 0.0 .TP -.B salt.modules.cluster.distrib(minions, master_conf, master_pem, conf_file) -Set up this minion as a failover master \- only intended for use by the -cluster interface -.UNINDENT -.SS salt.modules.cmd -.sp -A module for shelling out -.sp -Keep in mind that this module is insecure, in that it can give whomever has -access to the master root execution access to all salt minions -.INDENT 0.0 +.B salt.modules.git.checkout(cwd, rev, force=False, opts=None, user=None) +Checkout a given revision +.INDENT 7.0 .TP -.B salt.modules.cmd.exec_code(lang, code, cwd=None) -Pass in two strings, the first naming the executable language, aka \- -python2, python3, ruby, perl, lua, etc. the second string containing -the code you wish to execute. The stdout and stderr will be returned +.B cwd +The path to the Git repository +.TP +.B rev +The remote branch or revision to checkout +.TP +.B force +False +Force a checkout even if there might be overwritten changes +.TP +.B opts +None +Any additional options to add to the command line +.TP +.B user +None +Run git as a user other than what the minion runs as +.UNINDENT .sp -CLI Example: +CLI Examples: .sp .nf .ft C -salt \(aq*\(aq cmd.exec_code ruby \(aqputs "cheese"\(aq +salt \(aq*\(aq git.checkout /path/to/repo somebranch user=jeff + +salt \(aq*\(aq git.checkout /path/to/repo opts=\(aqtestbranch \-\- conf/file1 file2\(aq + +salt \(aq*\(aq git.checkout /path/to/repo rev=origin/mybranch opts=\-\-track .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cmd.has_exec(cmd) -Returns true if the executable is available on the minion, false otherwise +.B salt.modules.git.clone(cwd, repository, opts=None, user=None) +Clone a new repository +.INDENT 7.0 +.TP +.B cwd +The path to the Git repository +.TP +.B repository +The git uri of the repository +.TP +.B opts +None +Any additional options to add to the command line +.TP +.B user +None +Run git as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cmd.has_exec cat +salt \(aq*\(aq git.clone /path/to/repo git://github.com/saltstack/salt.git + +salt \(aq*\(aq git.clone /path/to/repo.git\e + git://github.com/saltstack/salt.git \(aq\-\-bare \-\-origin github\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cmd.retcode(cmd, cwd=None, runas=None) -Execute a shell command and return the command\(aqs return code. +.B salt.modules.git.describe(cwd, rev=\(aqHEAD\(aq, user=None) +Returns the git describe string (or the SHA hash if there are no tags) for +the given revision +.INDENT 7.0 +.TP +.B cwd +The path to the Git repository +.TP +.B rev: HEAD +The revision to describe +.TP +.B user +None +Run git as a user other than what the minion runs as +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.git.fetch(cwd, opts=None, user=None) +Perform a fetch on the given repository +.INDENT 7.0 +.TP +.B cwd +The path to the Git repository +.TP +.B opts +None +Any additional options to add to the command line +.TP +.B user +None +Run git as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cmd.retcode "file /bin/bash" +salt \(aq*\(aq git.fetch /path/to/repo \(aq\-\-all\(aq + +salt \(aq*\(aq git.fetch cwd=/path/to/repo opts=\(aq\-\-all\(aq user=johnny .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cmd.run(cmd, cwd=None, runas=None) -Execute the passed command and return the output as a string +.B salt.modules.git.init(cwd, opts=None, user=None) +Initialize a new git repository +.INDENT 7.0 +.TP +.B cwd +The path to the Git repository +.TP +.B opts +None +Any additional options to add to the command line +.TP +.B user +None +Run git as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cmd.run "ls \-l | awk \(aq/foo/{print $2}\(aq" +salt \(aq*\(aq git.init /path/to/repo.git opts=\(aq\-\-bare\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cmd.run_all(cmd, cwd=None, runas=None) -Execute the passed command and return a dict of return data +.B salt.modules.git.merge(cwd, branch=\(aq@{upstream}\(aq, opts=None, user=None) +Merge a given branch +.INDENT 7.0 +.TP +.B cwd +The path to the Git repository +.TP +.B branch +@{upstream} +The remote branch or revision to merge into the current branch +.TP +.B opts +None +Any additional options to add to the command line +.TP +.B user +None +Run git as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cmd.run_all "ls \-l | awk \(aq/foo/{print $2}\(aq" +salt \(aq*\(aq git.fetch /path/to/repo +salt \(aq*\(aq git.merge /path/to/repo @{upstream} .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cmd.run_stderr(cmd, cwd=None, runas=None) -Execute a command and only return the standard error +.B salt.modules.git.pull(cwd, opts=None, user=None) +Perform a pull on the given repository +.INDENT 7.0 +.TP +.B cwd +The path to the Git repository +.TP +.B opts +None +Any additional options to add to the command line +.TP +.B user +None +Run git as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cmd.run_stderr "ls \-l | awk \(aq/foo/{print $2}\(aq" +salt \(aq*\(aq git.pull /path/to/repo opts=\(aq\-\-rebase origin master\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cmd.run_stdout(cmd, cwd=None, runas=None) -Execute a command, and only return the standard out +.B salt.modules.git.rebase(cwd, rev=\(aqmaster\(aq, opts=None, user=None) +Rebase the current branch +.INDENT 7.0 +.TP +.B cwd +The path to the Git repository +.TP +.B rev +master +The revision to rebase onto the current branch +.TP +.B opts +None +Any additional options to add to the command line +.TP +.B user +None +Run git as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cmd.run_stdout "ls \-l | awk \(aq/foo/{print $2}\(aq" +salt \(aq*\(aq git.rebase /path/to/repo master .ft P .fi +.sp +That is the same as: git rebase master .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cmd.which(cmd) -Returns the path of an executable available on the minion, None otherwise +.B salt.modules.git.revision(cwd, rev=\(aqHEAD\(aq, short=False, user=None) +Returns the long hash of a given identifier (hash, branch, tag, HEAD, etc) +.INDENT 7.0 +.TP +.B cwd +The path to the Git repository +.TP +.B rev: HEAD +The revision +.TP +.B short: False +Return an abbreviated SHA1 git hash +.TP +.B user +None +Run git as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cmd.which cat +salt \(aq*\(aq git.revision /path/to/repo mybranch .ft P .fi .UNINDENT -.SS salt.modules.cp +.SS salt.modules.grains .sp -Minion side functions for salt\-cp +Control aspects of the grains data .INDENT 0.0 .TP -.B salt.modules.cp.cache_dir(path, env=\(aqbase\(aq) -Download and cache everything under a directory from the master +.B salt.modules.grains.item(key=None) +Return a singe component of the grains data .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cp.cache_dir salt://path/to/dir +salt \(aq*\(aq grains.item os .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cp.cache_file(path, env=\(aqbase\(aq) -Used to cache a single file in the local salt\-master file cache. +.B salt.modules.grains.items() +Return the grains data .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cp.cache_file salt://path/to/file +salt \(aq*\(aq grains.items .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cp.cache_files(paths, env=\(aqbase\(aq) -Used to gather many files from the master, the gathered files will be -saved in the minion cachedir reflective to the paths retrieved from the -master. +.B salt.modules.grains.ls() +Return a list of all available grains .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cp.cache_files salt://pathto/file1,salt://pathto/file1 +salt \(aq*\(aq grains.ls .ft P .fi .UNINDENT +.SS salt.modules.groupadd +.sp +Manage groups on Linux .INDENT 0.0 .TP -.B salt.modules.cp.cache_local_file(path) -Cache a local file on the minion in the localfiles cache +.B salt.modules.groupadd.add(name, gid=None) +Add the specified group .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cp.cache_local_file /etc/hosts +salt \(aq*\(aq group.add foo 3456 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cp.cache_master(env=\(aqbase\(aq) -Retrieve all of the files on the master and cache them locally +.B salt.modules.groupadd.chgid(name, gid) +Change the gid for a named group .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cp.cache_master +salt \(aq*\(aq group.chgid foo 4376 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cp.get_dir(path, dest, env=\(aqbase\(aq) -Used to recursively copy a directory from the salt master +.B salt.modules.groupadd.delete(name) +Remove the named group .sp CLI Example: -.INDENT 7.0 -.INDENT 3.5 -salt \(aq*\(aq cp.get_dir salt://path/to/dir/ /minion/dest -.UNINDENT -.UNINDENT +.sp +.nf +.ft C +salt \(aq*\(aq group.delete foo +.ft P +.fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cp.get_file(path, dest, env=\(aqbase\(aq) -Used to get a single file from the salt master +.B salt.modules.groupadd.getent() +Return info on all groups .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cp.get_file salt://path/to/file /minion/dest +salt \(aq*\(aq group.getent .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cp.get_url(path, dest, env=\(aqbase\(aq) -Used to get a single file from a URL. +.B salt.modules.groupadd.info(name) +Return information about a group .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cp.get_url salt://my/file /tmp/mine -salt \(aq*\(aq cp.get_url http://www.slashdot.org /tmp/index.html +salt \(aq*\(aq group.info foo .ft P .fi .UNINDENT +.SS salt.modules.hg +.sp +Support for the Mercurial SCM .INDENT 0.0 .TP -.B salt.modules.cp.hash_file(path, env=\(aqbase\(aq) -Return the hash of a file, to get the hash of a file on the -salt master file server prepend the path with salt://<file on server> -otherwise, prepend the file with / for a local file. +.B salt.modules.hg.archive(cwd, output, rev=\(aqtip\(aq, fmt=None, prefix=None, user=None) +Export a tarball from the repository +.INDENT 7.0 +.TP +.B cwd +The path to the Mercurial repository +.TP +.B output +The path to the archive tarball +.TP +.B rev: tip +The revision to create an archive from +.TP +.B fmt: None +Format of the resulting archive. Mercurial supports: tar, +tbz2, tgz, zip, uzip, and files formats. +.TP +.B prefix +None +Prepend <prefix>/ to every filename in the archive +.TP +.B user +None +Run hg as a user other than what the minion runs as +.UNINDENT +.sp +If \fBprefix\fP is not specified it defaults to the basename of the repo +directory. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cp.hash_file salt://path/to/file +salt \(aq*\(aq hg.archive /path/to/repo output=/tmp/archive.tgz fmt=tgz .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cp.is_cached(path, env=\(aqbase\(aq) -Return a boolean if the given path on the master has been cached on the -minion +.B salt.modules.hg.clone(cwd, repository, opts=None, user=None) +Clone a new repository +.INDENT 7.0 +.TP +.B cwd +The path to the Mercurial repository +.TP +.B repository +The hg uri of the repository +.TP +.B opts +None +Any additional options to add to the command line +.TP +.B user +None +Run hg as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cp.is_cached salt://path/to/file +salt \(aq*\(aq hg.clone /path/to/repo https://bitbucket.org/birkenfeld/sphinx .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cp.list_master(env=\(aqbase\(aq) -List all of the files stored on the master +.B salt.modules.hg.describe(cwd, rev=\(aqtip\(aq, user=None) +Mimick git describe and return an identifier for the given revision +.INDENT 7.0 +.TP +.B cwd +The path to the Mercurial repository +.TP +.B rev: tip +The path to the archive tarball +.TP +.B user +None +Run hg as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cp.list_master +salt \(aq*\(aq hg.describe /path/to/repo .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cp.list_minion(env=\(aqbase\(aq) -List all of the files cached on the minion +.B salt.modules.hg.pull(cwd, opts=None, user=None) +Perform a pull on the given repository +.INDENT 7.0 +.TP +.B cwd +The path to the Mercurial repository +.TP +.B opts +None +Any additional options to add to the command line +.TP +.B user +None +Run hg as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cp.list_minion +salt \(aq*\(aq hg.pull /path/to/repo \(aq\-u\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cp.recv(files, dest) -Used with salt\-cp, pass the files dict, and the destination. -.sp -This function receives small fast copy files from the master via salt\-cp -.UNINDENT -.SS salt.modules.cron -.sp -Work with cron -.INDENT 0.0 +.B salt.modules.hg.revision(cwd, rev=\(aqtip\(aq, short=False, user=None) +Returns the long hash of a given identifier (hash, branch, tag, HEAD, etc) +.INDENT 7.0 .TP -.B salt.modules.cron.list_tab(user) -Return the contents of the specified user\(aqs crontab +.B cwd +The path to the Mercurial repository +.TP +.B rev: tip +The revision +.TP +.B short: False +Return an abbreviated commit hash +.TP +.B user +None +Run hg as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cron.list_tab root +salt \(aq*\(aq hg.revision /path/to/repo mybranch .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cron.ls(user) -Return the contents of the specified user\(aqs crontab +.B salt.modules.hg.update(cwd, rev, force=False, user=None) +Update to a given revision +.INDENT 7.0 +.TP +.B cwd +The path to the Mercurial repository +.TP +.B rev +The revision to update to +.TP +.B force +False +Force an update +.TP +.B user +None +Run hg as a user other than what the minion runs as +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq cron.list_tab root +salt devserver1 hg.update /path/to/repo somebranch .ft P .fi .UNINDENT +.SS salt.modules.hosts +.sp +Manage the information in the hosts file .INDENT 0.0 .TP -.B salt.modules.cron.raw_cron(user) -Return the contents of the user\(aqs crontab +.B salt.modules.hosts.add_host(ip, alias) +Add a host to an existing entry, if the entry is not in place then create +it with the given host +.INDENT 7.0 +.TP +.B CLI Example:: +salt \(aq*\(aq hosts.add_host <ip> <alias> +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cron.rm(user, minute, hour, dom, month, dow, cmd) -Remove a cron job up for a specified user. +.B salt.modules.hosts.get_alias(ip) +Return the list of aliases associated with an ip +.INDENT 7.0 +.TP +.B CLI Example:: +salt \(aq*\(aq hosts.get_alias <ip addr> +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cron.rm_job(user, minute, hour, dom, month, dow, cmd) -Remove a cron job up for a specified user. +.B salt.modules.hosts.get_ip(host) +Return the ip associated with the named host +.INDENT 7.0 +.TP +.B CLI Example:: +salt \(aq*\(aq hosts.get_ip <hostname> +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cron.set_job(user, minute, hour, dom, month, dow, cmd) -Sets a cron job up for a specified user. +.B salt.modules.hosts.has_pair(ip, alias) +Return true if the alias is set +.INDENT 7.0 +.TP +.B CLI Example:: +salt \(aq*\(aq hosts.has_pair <ip> <alias> +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.cron.set_special(user, special, cmd) -Set up a special command in the crontab. -.sp -CLI Example: +.B salt.modules.hosts.list_hosts() +Return the hosts found in the hosts file in this format: .sp .nf .ft C -salt \(aq*\(aq cron.set_special @hourly \(aqecho foobar\(aq +{\(aq<ip addr>\(aq: [\(aqalias1\(aq, \(aqalias2\(aq, ...]} .ft P .fi -.UNINDENT -.SS salt.modules.data -.sp -Manage a local persistent data structure that can hold any arbitrairy data -specific to the minion -.INDENT 0.0 -.TP -.B salt.modules.data.dump(new_data) -Replace the entire datastore with a passed data structure .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq data.dump \(aq{\(aqeggs\(aq: \(aqspam\(aq}\(aq +salt \(aq*\(aq hosts.list_hosts .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.data.getval(key) -Get a value from the minion datastore -.sp -CLI Example: -.sp -.nf -.ft C -salt \(aq*\(aq data.getval <key> -.ft P -.fi +.B salt.modules.hosts.rm_host(ip, alias) +Remove a host entry from the hosts file +.INDENT 7.0 +.TP +.B CLI Example:: +salt \(aq*\(aq hosts.rm_host <ip> <alias> +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.data.getvals(keys) -Get values from the minion datastore +.B salt.modules.hosts.set_host(ip, alias) +Set the host entry in the hosts file for the given ip, this will overwrite +any previous entry for the given ip +.INDENT 7.0 +.TP +.B CLI Example:: +salt \(aq*\(aq hosts.set_host <ip> <alias> +.UNINDENT +.UNINDENT +.SS salt.modules.kmod +.sp +Module to manage Linux kernel modules +.INDENT 0.0 +.TP +.B salt.modules.kmod.available() +Return a list of all available kernel modules .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq data.getvals <key> +salt \(aq*\(aq kmod.available .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.data.load() -Return all of the data in the minion datastore +.B salt.modules.kmod.check_available(mod) +Check to see if the specified kernel module is available .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq data.load +salt \(aq*\(aq kmod.check_available kvm .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.data.update(key, value) -Update a key with a value in the minion datastore +.B salt.modules.kmod.load(mod) +Load the specified kernel module .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq data.update <key> <value> +salt \(aq*\(aq kmod.load kvm .ft P .fi .UNINDENT -.SS salt.modules.disk -.sp -Module for gathering disk information .INDENT 0.0 .TP -.B salt.modules.disk.inodeusage() -Return inode usage information for volumes mounted on this minion +.B salt.modules.kmod.lsmod() +Return a dict containing information about currently loaded modules .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq disk.inodeusage +salt \(aq*\(aq kmod.lsmod .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.disk.usage() -Return usage information for volumes mounted on this minion +.B salt.modules.kmod.remove(mod) +Remove the specified kernel module .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq disk.usage +salt \(aq*\(aq kmod.remove kvm .ft P .fi .UNINDENT -.SS salt.modules.ebuild +.SS salt.modules.kvm_hyper .sp -Support for Portage +Provide the hyper module for kvm hypervisors. This is the interface used to +interact with kvm on behalf of the salt\-virt interface +.sp +Required python modules: libvirt .INDENT 0.0 .TP -.B salt.modules.ebuild.available_version(name) -The available version of the package in the repository +.B salt.modules.kvm_hyper.freecpu() +Return an int representing the number of unallocated cpus on this +hypervisor .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.available_version <package name> +salt \(aq*\(aq virt.freemem .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ebuild.install(pkg, refresh=False, **kwargs) -Install the passed package -.sp -Return a dict containing the new package names and versions: -.sp -.nf -.ft C -{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, - \(aqnew\(aq: \(aq<new\-version>\(aq]} -.ft P -.fi +.B salt.modules.kvm_hyper.freemem() +Return an int representing the amount of memory that has not been given +to virtual machines on this node .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.install <package name> +salt \(aq*\(aq virt.freemem .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ebuild.list_pkgs() -List the packages currently installed in a dict: -.sp -.nf -.ft C -{\(aq<package_name>\(aq: \(aq<version>\(aq} -.ft P -.fi +.B salt.modules.kvm_hyper.get_conf(name) +Returns the xml for a given vm .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.list_pkgs +salt \(aq*\(aq virt.get_conf <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ebuild.purge(pkg) -Portage does not have a purge, this function calls remove -.sp -Return a list containing the removed packages: +.B salt.modules.kvm_hyper.get_disks(name) +Return the disks of a named virt .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.purge <package name> +salt \(aq*\(aq virt.get_disks <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ebuild.refresh_db() -Updates the portage tree (emerge \-\-sync) +.B salt.modules.kvm_hyper.halt(name) +Hard power down a virtual machine +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.kvm_hyper.hyper_info() +Return a dict with information about this hypervisor .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.refresh_db +salt \(aq*\(aq virt.node_info .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ebuild.remove(pkg) -Remove a single package via emerge \-\-unmerge +.B salt.modules.kvm_hyper.hyper_type() +Return that type of hypervisor this is +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.kvm_hyper.init(name, cpus, mem, image, storage_dir, network={\(aqeth0\(aq: {\(aqbridge\(aq: \(aqbr0\(aq, \(aqmac\(aq: \(aq\(aq}}, desc=\(aq\(aq, opts={}) +Create a KVM virtual machine based on these passed options, the virtual +machine will be started upon creation .sp -Return a list containing the names of the removed packages: +CLI Example: +.INDENT 7.0 +.INDENT 3.5 +salt node1 webserver 2 2048 salt://fedora/f16.img:virt /srv/vm/images +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.kvm_hyper.list_virts() +Return a list of virtual machine names on the minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.remove <package name> +salt \(aq*\(aq virt.list_virts .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ebuild.update(pkg, refresh=False) -Updates the passed package (emerge \-\-update package) -.sp -Return a dict containing the new package names and versions: +.B salt.modules.kvm_hyper.pause(name) +Pause the named virtual machine +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.kvm_hyper.purge(name) +Hard power down and purge a virtual machine, this will destroy a vm and +all associated vm data +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.kvm_hyper.resume(name) +Resume the named virtual machine +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.kvm_hyper.set_autostart(name) +Set the named virtual machine to autostart when the hypervisor boots +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.kvm_hyper.start(config) +Start an already defined virtual machine that has been shut down +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.kvm_hyper.virt_info() +Return detailed information about the vms on this hyper in a dict: .sp .nf .ft C -{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, - \(aqnew\(aq: \(aq<new\-version>\(aq]} +{\(aqcpu\(aq: <int>, +\(aqmaxMem\(aq: <int>, +\(aqmem\(aq: <int>, +\(aqstate\(aq: \(aq<state>\(aq, +\(aqcputime\(aq <int>} .ft P .fi .sp @@ -2563,2265 +6855,2530 @@ CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.update <package name> +salt \(aq*\(aq virt.vm_info .ft P .fi .UNINDENT +.SS salt.modules.linux_sysctl +.sp +Module for viewing and modifying sysctl parameters .INDENT 0.0 .TP -.B salt.modules.ebuild.upgrade(refresh=False) -Run a full system upgrade (emerge \-\-update world) -.sp -Return a dict containing the new package names and versions: +.B salt.modules.linux_sysctl.assign(name, value) +Assign a single sysctl parameter for this minion .sp -.nf -.ft C -{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, - \(aqnew\(aq: \(aq<new\-version>\(aq]} -.ft P -.fi +CLI Example: +salt \(aq*\(aq sysctl.assign net.ipv4.ip_forward 1 +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.linux_sysctl.get(name) +Return a single sysctl parameter for this minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.upgrade +salt \(aq*\(aq sysctl.get net.ipv4.ip_forward .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ebuild.version(name) -Returns a version if the package is installed, else returns an empty string +.B salt.modules.linux_sysctl.persist(name, value, config=\(aq/etc/sysctl.conf\(aq) +Assign and persist a simple sysctl parameter for this minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.version <package name> +salt \(aq*\(aq sysctl.persist net.ipv4.ip_forward 1 .ft P .fi .UNINDENT -.SS salt.modules.file -.sp -Manage information about files on the minion, set/read user, group, and mode -data .INDENT 0.0 .TP -.B salt.modules.file.append(path, *args) -Append text to the end of a file +.B salt.modules.linux_sysctl.show() +Return a list of sysctl parameters for this minion .sp -Usage: +CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.append /etc/motd \e - "With all thine offerings thou shalt offer salt."\e - "Salt is what makes things taste bad when it isn\(aqt in them." +salt \(aq*\(aq sysctl.show .ft P .fi -.sp -New in version 0.9.5. .UNINDENT +.SS salt.modules.mdadm +.sp +Salt module to manage RAID arrays with mdadm .INDENT 0.0 .TP -.B salt.modules.file.chgrp(path, group) -Change the group of a file +.B salt.modules.mdadm.detail(device=\(aq/dev/md0\(aq) +Show detail for a specified RAID device .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.chgrp /etc/passwd root +salt \(aq*\(aq raid.detail \(aq/dev/md0\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.chown(path, user, group) -Chown a file, pass the file the desired user and group +.B salt.modules.mdadm.list() +List the RAID devices. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.chown /etc/passwd root root +salt \(aq*\(aq raid.list .ft P .fi .UNINDENT +.SS salt.modules.moosefs +.sp +Module for gathering and managing information about MooseFS .INDENT 0.0 .TP -.B salt.modules.file.comment(path, regex, char=\(aq#\(aq, backup=\(aq.bak\(aq) -Comment out specified lines in a file -.INDENT 7.0 -.TP -.B path -The full path to the file to be edited -.TP -.B regex -A regular expression used to find the lines that are to be commented; -this pattern will be wrapped in parenthesis and will move any -preceding/trailing \fB^\fP or \fB$\fP characters outside the parenthesis -(e.g., the pattern \fB^foo$\fP will be rewritten as \fB^(foo)$\fP) -.TP -.B char -\fB#\fP -The character to be inserted at the beginning of a line in order to -comment it out -.TP -.B backup -\fB.bak\fP -The file will be backed up before edit with this file extension -.IP Warning -This backup will be overwritten each time \fBsed\fP / \fBcomment\fP / -\fBuncomment\fP is called. Meaning the backup will only be useful -after the first invocation. -.RE -.UNINDENT +.B salt.modules.moosefs.dirinfo(path, opts=None) +Return information on a directory located on the Moose .sp -Usage: +CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.comment /etc/modules pcspkr +salt \(aq*\(aq moosefs.dirinfo /path/to/dir/ [\-[n][h|H]] .ft P .fi -.sp -New in version 0.9.5. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.contains(path, text, limit=\(aq\(aq) -Return True if the file at \fBpath\fP contains \fBtext\fP +.B salt.modules.moosefs.fileinfo(path) +Return information on a file located on the Moose .sp -Usage: +CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.contains /etc/crontab \(aqmymaintenance.sh\(aq +salt \(aq*\(aq moosefs.fileinfo /path/to/dir/ .ft P .fi -.sp -New in version 0.9.5. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.find(path, *opts) -Approximate the Unix find(1) command and return a list of paths that -meet the specified critera. +.B salt.modules.moosefs.getgoal(path, opts=None) +Return goal(s) for a file or directory .sp -The options include match criteria: +CLI Example: .sp .nf .ft C -name = path\-glob # case sensitive -iname = path\-glob # case insensitive -regex = path\-regex # case sensitive -iregex = path\-regex # case insensitive -type = file\-types # match any listed type -user = users # match any listed user -group = groups # match any listed group -size = [+\-]number[size\-unit] # default unit = byte -mtime = interval # modified since date -grep = regex # search file contents +salt \(aq*\(aq moosefs.getgoal /path/to/file [\-[n][h|H]] +salt \(aq*\(aq moosefs.getgoal /path/to/dir/ [\-[n][h|H][r]] .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.moosefs.mounts() +Return a list of current MooseFS mounts .sp -and/or actions: +CLI Example: .sp .nf .ft C -delete [= file\-types] # default type = \(aqf\(aq -exec = command [arg ...] # where {} is replaced by pathname -print [= print\-opts] +salt \(aq*\(aq moosefs.mounts .ft P .fi +.UNINDENT +.SS salt.modules.mount .sp -The default action is \(aqprint=path\(aq. +Salt module to manage unix mounts and the fstab file +.INDENT 0.0 +.TP +.B salt.modules.mount.active() +List the active mounts. .sp -file\-glob: +CLI Example: .sp .nf .ft C -* = match zero or more chars -? = match any char -[abc] = match a, b, or c -[!abc] or [^abc] = match anything except a, b, and c -[x\-y] = match chars x through y -[!x\-y] or [^x\-y] = match anything except chars x through y -{a,b,c} = match a or b or c +salt \(aq*\(aq mount.active .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.mount.fstab(config=\(aq/etc/fstab\(aq) +List the contents of the fstab .sp -path\-regex: a Python re (regular expression) pattern to match pathnames -.sp -file\-types: a string of one or more of the following: +CLI Example: .sp .nf .ft C -a: all file types -b: block device -c: character device -d: directory -p: FIFO (named pipe) -f: plain file -l: symlink -s: socket +salt \(aq*\(aq mount.fstab .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.mount.is_fuse_exec(cmd) +Returns true if the command passed is a fuse mountable application. .sp -users: a space and/or comma separated list of user names and/or uids -.sp -groups: a space and/or comma separated list of group names and/or gids -.sp -size\-unit: +CLI Example: .sp .nf .ft C -b: bytes -k: kilobytes -m: megabytes -g: gigabytes -t: terabytes +salt \(aq*\(aq mount.is_fuse_exec sshfs .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.mount.mount(name, device, mkmnt=False, fstype=\(aq\(aq, opts=\(aqdefaults\(aq) +Mount a device .sp -interval: +CLI Example: .sp .nf .ft C -[<num>w] [<num>[d]] [<num>h] [<num>m] [<num>s] - -where: - w: week - d: day - h: hour - m: minute - s: second +salt \(aq*\(aq mount.mount /mnt/foo /dev/sdz1 True .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.mount.remount(name, device, mkmnt=False, fstype=\(aq\(aq, opts=\(aqdefaults\(aq) +Attempt to remount a device, if the device is not already mounted, mount +is called .sp -print\-opts: a comma and/or space separated list of one or more of the -following: +CLI Example: .sp .nf .ft C -group: group name -md5: MD5 digest of file contents -mode: file permissions (as integer) -mtime: last modification time (as time_t) -name: file basename -path: file absolute path -size: file size in bytes -type: file type -user: user name +salt \(aq*\(aq mount.remount /mnt/foo /dev/sdz1 True .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.mount.rm_fstab(name, config=\(aq/etc/fstab\(aq) +Remove the mount point from the fstab .sp -CLI Examples: +CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.find / type=f name=\e*.bak size=+10m -salt \(aq*\(aq file.find /var mtime=+30d size=+10m print=path,size,mtime -salt \(aq*\(aq file.find /var/log name=\e*.[0\-9] mtime=+30d size=+10m delete +salt \(aq*\(aq /mnt/foo .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.get_gid(path) -Return the id of the group that owns a given file +.B salt.modules.mount.set_fstab(name, device, fstype, opts=\(aqdefaults\(aq, dump=0, pass_num=0, config=\(aq/etc/fstab\(aq) +Verify that this mount is represented in the fstab, chage the mount point +to match the data passed, or add the mount if it is not present. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.get_gid /etc/passwd +salt \(aq*\(aq mount.set_fstab /mnt/foo /dev/sdz1 ext4 .ft P .fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.file.get_group(path) -Return the group that owns a given file +.SS salt.modules.mysql .sp -CLI Example: +Module to provide MySQL compatibility to salt. +.sp +In order to connect to MySQL, certain configuration is required +in /etc/salt/minion on the relevant minions. Some sample configs +might look like: .sp .nf .ft C -salt \(aq*\(aq file.get_group /etc/passwd +mysql.host: \(aqlocalhost\(aq +mysql.port: 3306 +mysql.user: \(aqroot\(aq +mysql.pass: \(aq\(aq +mysql.db: \(aqmysql\(aq .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.file.get_mode(path) -Return the mode of a file .sp -CLI Example: +You can also use a defaults file: .sp .nf .ft C -salt \(aq*\(aq file.get_mode /etc/passwd +mysql.default_file: \(aq/etc/mysql/debian.cnf\(aq .ft P .fi +.sp +Required python modules: MySQLdb +.INDENT 0.0 +.TP +.B salt.modules.mysql.connect(**kwargs) +wrap authentication credentials here .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.get_sum(path, form=\(aqmd5\(aq) -Return the sum for the given file, default is md5, sha1, sha224, sha256, -sha384, sha512 are supported +.B salt.modules.mysql.db_check(name, table=None) +Repairs the full database or just a given table .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.get_sum /etc/passwd sha512 +salt \(aq*\(aq mysql.db_check dbname .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.get_uid(path) -Return the id of the user that owns a given file +.B salt.modules.mysql.db_create(name) +Adds a databases to the MySQL server. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.get_uid /etc/passwd +salt \(aq*\(aq mysql.db_create \(aqdbname\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.get_user(path) -Return the user that owns a given file +.B salt.modules.mysql.db_exists(name) +Checks if a database exists on the MySQL server. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.get_user /etc/passwd +salt \(aq*\(aq mysql.db_exists \(aqdbname\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.gid_to_group(gid) -Convert the group id to the group name on this system +.B salt.modules.mysql.db_list() +Return a list of databases of a MySQL server using the output +from the \fBSHOW DATABASES\fP query. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.gid_to_group 0 +salt \(aq*\(aq mysql.db_list .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.group_to_gid(group) -Convert the group to the gid on this system +.B salt.modules.mysql.db_optimize(name, table=None) +Optimizes the full database or just a given table .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.group_to_gid root +salt \(aq*\(aq mysql.db_optimize dbname .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.sed(path, before, after, limit=\(aq\(aq, backup=\(aq.bak\(aq, options=\(aq\-r \-e\(aq, flags=\(aqg\(aq) -Make a simple edit to a file +.B salt.modules.mysql.db_remove(name) +Removes a databases from the MySQL server. .sp -Equivalent to: +CLI Example: .sp .nf .ft C -sed <backup> <options> "/<limit>/ s/<before>/<after>/<flags> <file>" +salt \(aq*\(aq mysql.db_remove \(aqdbname\(aq .ft P .fi -.INDENT 7.0 -.TP -.B path -The full path to the file to be edited -.TP -.B before -A pattern to find in order to replace with \fBafter\fP -.TP -.B after -Text that will replace \fBbefore\fP -.TP -.B limit -\fB\(aq\(aq\fP -An initial pattern to search for before searching for \fBbefore\fP -.TP -.B backup -\fB.bak\fP -The file will be backed up before edit with this file extension; -\fBWARNING:\fP each time \fBsed\fP/\fBcomment\fP/\fBuncomment\fP is called will -overwrite this backup -.TP -.B options -\fB\-r \-e\fP -Options to pass to sed -.TP -.B flags -\fBg\fP -Flags to modify the sed search; e.g., \fBi\fP for case\-insensitve pattern -matching .UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.mysql.db_repair(name, table=None) +Repairs the full database or just a given table .sp -Forward slashes and single quotes will be escaped automatically in the -\fBbefore\fP and \fBafter\fP patterns. -.sp -Usage: +CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.sed /etc/httpd/httpd.conf \(aqLogLevel warn\(aq \(aqLogLevel info\(aq +salt \(aq*\(aq mysql.db_repair dbname .ft P .fi -.sp -New in version 0.9.5. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.set_mode(path, mode) -Set the mode of a file +.B salt.modules.mysql.db_tables(name) +Shows the tables in the given MySQL database (if exists) .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.set_mode /etc/passwd 0644 +salt \(aq*\(aq mysql.db_tables \(aqdatabase\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.touch(name, atime=None, mtime=None) -Just like \(aqnix\(aqs "touch" command, create a file if it -doesn\(aqt exist or simply update the atime and mtime if -it already does. -.INDENT 7.0 -.TP -.B atime: -Access time in Unix epoch time -.TP -.B mtime: -Last modification in Unix epoch time -.TP -.B Usage:: -salt \(aq*\(aq file.touch /var/log/emptyfile -.UNINDENT -.sp -New in version 0.9.5. +.B salt.modules.mysql.free_slave() +Frees a slave from its master. This is a WIP, do not use. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.uid_to_user(uid) -Convert a uid to a user name +.B salt.modules.mysql.grant_add(grant, database, user, host=\(aqlocalhost\(aq, grant_option=False, escape=True) +Adds a grant to the MySQL server. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.uid_to_user 0 +salt \(aq*\(aq mysql.grant_add \(aqSELECT|INSERT|UPDATE|...\(aq \(aqdatabase.*\(aq \(aqfrank\(aq \(aqlocalhost\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.uncomment(path, regex, char=\(aq#\(aq, backup=\(aq.bak\(aq) -Uncomment specified commented lines in a file -.INDENT 7.0 -.TP -.B path -The full path to the file to be edited -.TP -.B regex -A regular expression used to find the lines that are to be uncommented. -This regex should not include the comment character. A leading \fB^\fP -character will be stripped for convenience (for easily switching -between comment() and uncomment()). -.TP -.B char -\fB#\fP -The character to remove in order to uncomment a line; if a single -whitespace character follows the comment it will also be removed -.TP -.B backup -\fB.bak\fP -The file will be backed up before edit with this file extension; -\fBWARNING:\fP each time \fBsed\fP/\fBcomment\fP/\fBuncomment\fP is called will -overwrite this backup -.UNINDENT +.B salt.modules.mysql.grant_revoke(grant, database, user, host=\(aqlocalhost\(aq, grant_option=False) +Removes a grant from the MySQL server. .sp -Usage: +CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.uncomment /etc/hosts.deny \(aqALL: PARANOID\(aq +salt \(aq*\(aq mysql.grant_revoke \(aqSELECT,INSERT,UPDATE\(aq \(aqdatabase.*\(aq \(aqfrank\(aq \(aqlocalhost\(aq .ft P .fi -.sp -New in version 0.9.5. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.file.user_to_uid(user) -Convert user name to a uid +.B salt.modules.mysql.slave_lag() +Return the number of seconds that a slave SQL server is lagging behind the +master, if the host is not a slave it will return \-1. If the server is +configured to be a slave for replication but slave IO is not running then +\-2 will be returned. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq file.user_to_uid root +salt \(aq*\(aq mysql.slave_lag .ft P .fi .UNINDENT -.SS salt.modules.freebsdkmod -.sp -Module to manage FreeBSD kernel modules .INDENT 0.0 .TP -.B salt.modules.freebsdkmod.available() -Return a list of all available kernel modules +.B salt.modules.mysql.status() +Return the status of a MySQL server using the output +from the \fBSHOW STATUS\fP query. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq kmod.available +salt \(aq*\(aq mysql.status .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.freebsdkmod.check_available(mod) -Check to see if the specified kernel module is available +.B salt.modules.mysql.user_chpass(user, host=\(aqlocalhost\(aq, password=None, password_hash=None) +Change password for MySQL user .sp -CLI Example: +CLI Examples: .sp .nf .ft C -salt \(aq*\(aq kmod.check_available kvm +salt \(aq*\(aq mysql.user_chpass frank localhost newpassword + +salt \(aq*\(aq mysql.user_chpass frank localhost password_hash=\(aqhash\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.freebsdkmod.load(mod) -Load the specified kernel module +.B salt.modules.mysql.user_create(user, host=\(aqlocalhost\(aq, password=None, password_hash=None) +Creates a MySQL user. .sp -CLI Example: +CLI Examples: .sp .nf .ft C -salt \(aq*\(aq kmod.load kvm +salt \(aq*\(aq mysql.user_create \(aqusername\(aq \(aqhostname\(aq \(aqpassword + +salt \(aq*\(aq mysql.user_create \(aqusername\(aq \(aqhostname\(aq password_hash=\(aqhash\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.freebsdkmod.lsmod() -Return a dict containing information about currently loaded modules +.B salt.modules.mysql.user_exists(user, host=\(aqlocalhost\(aq) +Checks if a user exists on the MySQL server. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq kmod.lsmod +salt \(aq*\(aq mysql.user_exists \(aqusername\(aq \(aqhostname\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.freebsdkmod.remove(mod) -Remove the specified kernel module +.B salt.modules.mysql.user_grants(user, host=\(aqlocalhost\(aq) +Shows the grants for the given MySQL user (if it exists) .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq kmod.remove kvm +salt \(aq*\(aq mysql.user_grants \(aqfrank\(aq \(aqlocalhost\(aq .ft P .fi .UNINDENT -.SS salt.modules.freebsdpkg -.sp -Package support for FreeBSD .INDENT 0.0 .TP -.B salt.modules.freebsdpkg.available_version(name) -The available version of the package in the repository +.B salt.modules.mysql.user_info(user, host=\(aqlocalhost\(aq) +Get full info on a MySQL user .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.available_version <package name> +salt \(aq*\(aq mysql.user_info root localhost .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.freebsdpkg.install(name, **kwargs) -Install the passed package +.B salt.modules.mysql.user_list() +Return a list of users on a MySQL server .sp -Return a dict containing the new package names and versions: +CLI Example: .sp .nf .ft C -{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, - \(aqnew\(aq: \(aq<new\-version>\(aq]} +salt \(aq*\(aq mysql.user_list .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.mysql.user_remove(user, host=\(aqlocalhost\(aq) +Delete MySQL user .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.install <package name> +salt \(aq*\(aq mysql.user_remove frank localhost .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.freebsdpkg.list_pkgs() -List the packages currently installed as a dict: -.sp -.nf -.ft C -{\(aq<package_name>\(aq: \(aq<version>\(aq} -.ft P -.fi +.B salt.modules.mysql.version() +Return the version of a MySQL server using the output +from the \fBSELECT VERSION()\fP query. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.list_pkgs +salt \(aq*\(aq mysql.version .ft P .fi .UNINDENT +.SS salt.modules.network +.sp +Module for gathering and managing network information .INDENT 0.0 .TP -.B salt.modules.freebsdpkg.purge(name) -Remove a single package with pkg_delete -.sp -Returns a list containing the removed packages. +.B salt.modules.network.dig(host) +Performs a DNS lookup with dig .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.purge <package name> +salt \(aq*\(aq network.dig archlinux.org .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.freebsdpkg.refresh_db() -Update the ports tree with portsnap. If the ports tree does not exist it -will be downloaded and set up. +.B salt.modules.network.hwaddr(interface) +Returns the hwaddr for a given interface .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.refresh_db +salt \(aq*\(aq network.hwaddr eth0 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.freebsdpkg.remove(name) -Remove a single package with pkg_delete -.sp -Returns a list containing the removed packages. +.B salt.modules.network.interfaces() +Returns a dictionary of interfaces with various information about each +(up/down state, ip address, netmask, and hwaddr) .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.remove <package name> +salt \(aq*\(aq network.interfaces .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.freebsdpkg.upgrade() -Run a full system upgrade, a \fBfreebsd\-update fetch install\fP +.B salt.modules.network.ipaddr(interface) +Returns the IP address for a given interface .sp -Return a dict containing the new package names and versions: +CLI Example: .sp .nf .ft C -{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, - \(aqnew\(aq: \(aq<new\-version>\(aq]} +salt \(aq*\(aq network.ipaddr eth0 .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.network.isportopen(host, port) +Return status of a port .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.upgrade +salt \(aq*\(aq network.isportopen 127.0.0.1 22 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.freebsdpkg.version(name) -Returns a version if the package is installed, else returns an empty string +.B salt.modules.network.netmask(interface) +Returns the netmask for a given interface .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.version <package name> +salt \(aq*\(aq network.netmask eth0 .ft P .fi .UNINDENT -.SS salt.modules.gentoo_service -.sp -Top level package command wrapper, used to translate the os detected by the -grains to the correct service manager .INDENT 0.0 .TP -.B salt.modules.gentoo_service.disable(name) -Disable the named service to start at boot +.B salt.modules.network.netstat() +Return information on open ports and states .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.disable <service name> +salt \(aq*\(aq network.netstat .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.gentoo_service.disabled(name) -Return True if the named servioce is enabled, false otherwise +.B salt.modules.network.ping(host) +Performs a ping to a host .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.enabled <service name> +salt \(aq*\(aq network.ping archlinux.org .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.gentoo_service.enable(name) -Enable the named service to start at boot +.B salt.modules.network.traceroute(host) +Performs a traceroute to a 3rd party host .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.enable <service name> +salt \(aq*\(aq network.traceroute archlinux.org .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.gentoo_service.enabled(name) -Return True if the named servioce is enabled, false otherwise +.B salt.modules.network.up(interface) +Returns True if interface is up, otherwise returns False .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.enabled <service name> +salt \(aq*\(aq network.up eth0 .ft P .fi .UNINDENT +.SS salt.modules.nginx +.sp +Support for nginx .INDENT 0.0 .TP -.B salt.modules.gentoo_service.get_all() -Return all available boot services +.B salt.modules.nginx.signal(signal=None) +Signals httpd to start, restart, or stop. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.get_enabled +salt \(aq*\(aq nginx.signal reload .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.gentoo_service.get_disabled() -Return a set of services that are installed but disabled +.B salt.modules.nginx.version() +Return server version from nginx \-v .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.get_enabled +salt \(aq*\(aq nginx.version .ft P .fi .UNINDENT +.SS salt.modules.pacman +.sp +A module to wrap pacman calls, since Arch is the best +(\fI\%https://wiki.archlinux.org/index.php/Arch_is_the_best\fP) .INDENT 0.0 .TP -.B salt.modules.gentoo_service.get_enabled() -Return a list of service that are enabled on boot +.B salt.modules.pacman.available_version(name) +The available version of the package in the repository .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.get_enabled +salt \(aq*\(aq pkg.available_version <package name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.gentoo_service.restart(name) -Restart the named service +.B salt.modules.pacman.install(name, refresh=False, **kwargs) +Install the passed package, add refresh=True to install with an \-Sy .sp -CLI Example: +Return a dict containing the new package names and versions: .sp .nf .ft C -salt \(aq*\(aq service.restart <service name> +{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, + \(aqnew\(aq: \(aq<new\-version>\(aq]} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.gentoo_service.start(name) -Start the specified service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.start <service name> +salt \(aq*\(aq pkg.install <package name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.gentoo_service.status(name, sig=None) -Return the status for a service, returns the PID or an empty string if the -service is running or not, pass a signature to use to find the service via -ps -.sp -CLI Example: +.B salt.modules.pacman.list_pkgs() +List the packages currently installed as a dict: .sp .nf .ft C -salt \(aq*\(aq service.status <service name> [service signature] +{\(aq<package_name>\(aq: \(aq<version>\(aq} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.gentoo_service.stop(name) -Stop the specified service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.stop <service name> +salt \(aq*\(aq pkg.list_pkgs .ft P .fi .UNINDENT -.SS salt.modules.grains -.sp -Control aspects of the grains data .INDENT 0.0 .TP -.B salt.modules.grains.item(key=None) -Return a singe component of the grains data +.B salt.modules.pacman.purge(name) +Recursively remove a package and all dependencies which were installed +with it, this will call a \fBpacman \-Rsc\fP +.sp +Return a list containing the removed packages. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq grains.item os +salt \(aq*\(aq pkg.purge <package name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.grains.items() -Return the grains data +.B salt.modules.pacman.refresh_db() +Just run a \fBpacman \-Sy\fP, return a dict: +.sp +.nf +.ft C +{\(aq<database name>\(aq: Bool} +.ft P +.fi .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq grains.items +salt \(aq*\(aq pkg.refresh_db .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.grains.ls() -Return a list of all available grains +.B salt.modules.pacman.remove(name) +Remove a single package with \fBpacman \-R\fP +.sp +Return a list containing the removed packages. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq grains.ls +salt \(aq*\(aq pkg.remove <package name> .ft P .fi .UNINDENT -.SS salt.modules.groupadd -.sp -Manage groups on Linux .INDENT 0.0 .TP -.B salt.modules.groupadd.add(name, gid=None) -Add the specified group +.B salt.modules.pacman.upgrade() +Run a full system upgrade, a pacman \-Syu .sp -CLI Example: +Return a dict containing the new package names and versions: .sp .nf .ft C -salt \(aq*\(aq group.add foo 3456 +{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, + \(aqnew\(aq: \(aq<new\-version>\(aq]} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.groupadd.chgid(name, gid) -Change the gid for a named group .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq group.chgid foo 4376 +salt \(aq*\(aq pkg.upgrade .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.groupadd.delete(name) -Remove the named group +.B salt.modules.pacman.upgrade_available(name) +Check whether or not an upgrade is available for a given package .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq group.delete foo +salt \(aq*\(aq pkg.upgrade_available <package name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.groupadd.getent() -Return info on all groups +.B salt.modules.pacman.version(name) +Returns a version if the package is installed, else returns an empty string .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq group.getent +salt \(aq*\(aq pkg.version <package name> .ft P .fi .UNINDENT +.SS salt.modules.pillar +.sp +Extract the pillar data for this minion .INDENT 0.0 .TP -.B salt.modules.groupadd.info(name) -Return information about a group +.B salt.modules.pillar.data() +Returns the pillar derived from the configured pillar source. The pillar +source is derived from the file_client option in the minion config .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq group.info foo +salt \(aq*\(aq pillar.data .ft P .fi .UNINDENT -.SS salt.modules.hosts +.SS salt.modules.pip .sp -Manage the information in the hosts file +Install Python packages with pip to either the system or a virtualenv .INDENT 0.0 .TP -.B salt.modules.hosts.add_host(ip, alias) -Add a host to an existing entry, if the entry is not in place then create -it with the given host +.B salt.modules.pip.freeze(bin_env=None) +Return a list of installed packages either globally or in the specified +virtualenv .INDENT 7.0 .TP -.B CLI Example:: -salt \(aq*\(aq hosts.add_host <ip> <alias> +.B bin_env +path to pip bin or path to virtualenv. If doing an uninstall from +the system python and want to use a specific pip bin (pip\-2.7, +pip\-2.6, etc..) just specify the pip bin you want. +If uninstalling from a virtualenv, just use the path to the virtualenv +(/home/code/path/to/virtualenv/) .UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.hosts.get_alias(ip) -Return the list of aliases associated with an ip +.B salt.modules.pip.install(pkgs=None, requirements=None, env=None, bin_env=None, log=None, proxy=None, timeout=None, editable=None, find_links=None, index_url=None, extra_index_url=None, no_index=False, mirrors=None, build=None, target=None, download=None, download_cache=None, source=None, upgrade=False, force_reinstall=False, ignore_installed=False, no_deps=False, no_install=False, no_download=False, install_options=None) +Install packages with pip +.sp +Install packages individually or from a pip requirements file. Install +packages globally or to a virtualenv. .INDENT 7.0 .TP -.B CLI Example:: -salt \(aq*\(aq hosts.get_alias <ip addr> -.UNINDENT -.UNINDENT -.INDENT 0.0 +.B pkgs +comma separated list of packages to install .TP -.B salt.modules.hosts.get_ip(host) -Return the ip associated with the named host -.INDENT 7.0 +.B requirements +path to requirements .TP -.B CLI Example:: -salt \(aq*\(aq hosts.get_ip <hostname> -.UNINDENT -.UNINDENT -.INDENT 0.0 +.B bin_env +path to pip bin or path to virtualenv. If doing a system install, +and want to use a specific pip bin (pip\-2.7, pip\-2.6, etc..) just +specify the pip bin you want. +If installing into a virtualenv, just use the path to the virtualenv +(/home/code/path/to/virtualenv/) .TP -.B salt.modules.hosts.has_pair(ip, alias) -Return true if the alias is set -.INDENT 7.0 +.B env +depreicated, use bin_env now .TP -.B CLI Example:: -salt \(aq*\(aq hosts.has_pair <ip> <alias> -.UNINDENT -.UNINDENT -.INDENT 0.0 +.B log +Log file where a complete (maximum verbosity) record will be kept .TP -.B salt.modules.hosts.list_hosts() -Return the hosts found in the hosts file in this format: +.B proxy +Specify a proxy in the form +user:passwd@proxy.server:port. Note that the +user:password@ is optional and required only if you +are behind an authenticated proxy. If you provide +user@proxy.server:port then you will be prompted for a +password. +.TP +.B timeout +Set the socket timeout (default 15 seconds) +.TP +.B editable +install something editable(ie git+https://github.com/worldcompany/djangoembed.git#egg=djangoembed) +.TP +.B find_links +URL to look for packages at +.TP +.B index_url +Base URL of Python Package Index +.TP +.B extra_index_url +Extra URLs of package indexes to use in addition to \fBindex_url\fP +.TP +.B no_index +Ignore package index +.TP +.B mirrors +Specific mirror URLs to query (automatically adds \-\-use\-mirrors) +.TP +.B build +Unpack packages into \fBbuild\fP dir +.TP +.B target +Install packages into \fBtarget\fP dir +.TP +.B download +Download packages into \fBdownload\fP instead of installing them +.TP +.B download_cache +Cache downloaded packages in \fBdownload_cache\fP dir +.TP +.B source +Check out \fBeditable\fP packages into \fBsource\fP dir +.TP +.B upgrade +Upgrade all packages to the newest available version +.TP +.B force_reinstall +When upgrading, reinstall all packages even if they are already up\-to\-date. +.TP +.B ignore_installed +Ignore the installed packages (reinstalling instead) +.TP +.B no_deps +Ignore package dependencies +.TP +.B no_install +Download and unpack all packages, but don\(aqt actually install them +.TP +.B no_download +Don\(aqt download any packages, just install the ones +already downloaded (completes an install run with +\-\-no\-install) +.TP +.B install_options +Extra arguments to be supplied to the setup.py install +command (use like \-\-install\-option="\-\-install\- +scripts=/usr/local/bin"). Use multiple \-\-install\- +option options to pass multiple options to setup.py +install. If you are using an option with a directory +path, be sure to use absolute path. +.UNINDENT +.sp +CLI Example: .sp .nf .ft C -{\(aq<ip addr>\(aq: [\(aqalias1\(aq, \(aqalias2\(aq, ...]} +salt \(aq*\(aq pip.install <package name>,<package2 name> + +salt \(aq*\(aq pip.install requirements=/path/to/requirements.txt + +salt \(aq*\(aq pip.install <package name> bin_env=/path/to/virtualenv + +salt \(aq*\(aq pip.install <package name> bin_env=/path/to/pip_bin .ft P .fi .sp -CLI Example: +Comlicated CLI example: .sp .nf .ft C -salt \(aq*\(aq hosts.list_hosts +salt \(aq*\(aq pip.install markdown,django editable=git+https://github.com/worldcompany/djangoembed.git#egg=djangoembed upgrade=True no_deps=True .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.hosts.rm_host(ip, alias) -Remove a host entry from the hosts file -.INDENT 7.0 -.TP -.B CLI Example:: -salt \(aq*\(aq hosts.rm_host <ip> <alias> -.UNINDENT +.B salt.modules.pip.list(prefix=\(aq\(aq, bin_env=None) +Filter list of instaslled apps from \fBfreeze\fP and check to see if \fBprefix\fP +exists in the list of packages installed. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.hosts.set_host(ip, alias) -Set the host entry in the hosts file for the given ip, this will overwrite -any previous entry for the given ip +.B salt.modules.pip.uninstall(pkgs=None, requirements=None, bin_env=None, log=None, proxy=None, timeout=None) +Uninstall packages with pip +.sp +Uninstall packages individually or from a pip requirements file. Uninstall +packages globally or from a virtualenv. .INDENT 7.0 .TP -.B CLI Example:: -salt \(aq*\(aq hosts.set_host <ip> <alias> +.B pkgs +comma separated list of packages to install +.TP +.B requirements +path to requirements +.TP +.B bin_env +path to pip bin or path to virtualenv. If doing an uninstall from +the system python and want to use a specific pip bin (pip\-2.7, +pip\-2.6, etc..) just specify the pip bin you want. +If uninstalling from a virtualenv, just use the path to the virtualenv +(/home/code/path/to/virtualenv/) +.TP +.B log +Log file where a complete (maximum verbosity) record will be kept +.TP +.B proxy +Specify a proxy in the form +user:passwd@proxy.server:port. Note that the +user:password@ is optional and required only if you +are behind an authenticated proxy. If you provide +user@proxy.server:port then you will be prompted for a +password. +.TP +.B timeout +Set the socket timeout (default 15 seconds) .UNINDENT +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pip.uninstall <package name>,<package2 name> + +salt \(aq*\(aq pip.uninstall requirements=/path/to/requirements.txt + +salt \(aq*\(aq pip.uninstall <package name> bin_env=/path/to/virtualenv + +salt \(aq*\(aq pip.uninstall <package name> bin_env=/path/to/pip_bin +.ft P +.fi .UNINDENT -.SS salt.modules.kmod +.SS salt.modules.ps .sp -Module to manage Linux kernel modules +A salt interface to psutil, a system and process library. +See \fI\%http://code.google.com/p/psutil\fP. +.sp +Required python modules: psutil .INDENT 0.0 .TP -.B salt.modules.kmod.available() -Return a list of all available kernel modules +.B salt.modules.ps.boot_time() +Return the boot time in number of seconds since the epoch began. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq kmod.available +salt \(aq*\(aq ps.boot_time .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.kmod.check_available(mod) -Check to see if the specified kernel module is available +.B salt.modules.ps.cached_physical_memory() +Return the amount cached memory. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq kmod.check_available kvm +salt \(aq*\(aq ps.cached_physical_memory .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.kmod.load(mod) -Load the specified kernel module +.B salt.modules.ps.cpu_percent(interval=0.1, per_cpu=False) +Return the percent of time the CPU is busy. +.INDENT 7.0 +.TP +.B interval +the number of seconds to sample CPU usage over +.TP +.B per_cpu +if True return an array of CPU percent busy for each CPU, otherwise +aggregate all percents into one number +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq kmod.load kvm +salt \(aq*\(aq ps.cpu_percent .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.kmod.lsmod() -Return a dict containing information about currently loaded modules +.B salt.modules.ps.cpu_times(per_cpu=False) +Return the percent of time the CPU spends in each state, +e.g. user, system, idle, nice, iowait, irq, softirq. +.INDENT 7.0 +.TP +.B per_cpu +if True return an array of percents for each CPU, otherwise aggregate +all percents into one number +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq kmod.lsmod +salt \(aq*\(aq ps.cpu_times .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.kmod.remove(mod) -Remove the specified kernel module +.B salt.modules.ps.disk_io_counters() +Return disk I/O statisitics. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq kmod.remove kvm +salt \(aq*\(aq ps.disk_io_counters .ft P .fi .UNINDENT -.SS salt.modules.linux_sysctl -.sp -Module for viewing and modifying sysctl parameters .INDENT 0.0 .TP -.B salt.modules.linux_sysctl.assign(name, value) -Assign a single sysctl parameter for this minion +.B salt.modules.ps.disk_partition_usage(all=False) +Return a list of disk partitions plus the mount point, filesystem and usage +statistics. .sp CLI Example: -salt \(aq*\(aq sysctl.assign net.ipv4.ip_forward 1 +.sp +.nf +.ft C +salt \(aq*\(aq ps.disk_partition_usage +.ft P +.fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.linux_sysctl.get(name) -Return a single sysctl parameter for this minion +.B salt.modules.ps.disk_partitions(all=False) +Return a list of disk partitions and their device, mount point, and +filesystem type. +.INDENT 7.0 +.TP +.B all +if set to False, only return local, physical partitions (hard disk, +USB, CD/DVD partitions). If True, return all filesystems. +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq sysctl.get net.ipv4.ip_forward +salt \(aq*\(aq ps.disk_partitions .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.linux_sysctl.persist(name, value, config=\(aq/etc/sysctl.conf\(aq) -Assign and persist a simple sysctl parameter for this minion +.B salt.modules.ps.disk_usage(path) +Given a path, return a dict listing the total available space as well as +the free space, and used space. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq sysctl.persist net.ipv4.ip_forward 1 +salt \(aq*\(aq ps.disk_usage /home .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.linux_sysctl.show() -Return a list of sysctl parameters for this minion +.B salt.modules.ps.get_pid_list() +Return a list of process ids (PIDs) for all running processes. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq sysctl.show +salt \(aq*\(aq ps.get_pid_list .ft P .fi .UNINDENT -.SS salt.modules.mdadm -.sp -Salt module to manage RAID arrays with mdadm .INDENT 0.0 .TP -.B salt.modules.mdadm.detail(device=\(aq/dev/md0\(aq) -Show detail for a specified RAID device +.B salt.modules.ps.network_io_counters() +Return network I/O statisitics. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq raid.detail \(aq/dev/md0\(aq +salt \(aq*\(aq ps.network_io_counters .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mdadm.list() -List the RAID devices. +.B salt.modules.ps.num_cpus() +Return the number of CPUs. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq raid.list +salt \(aq*\(aq ps.num_cpus .ft P .fi .UNINDENT -.SS salt.modules.moosefs -.sp -Module for gathering and managing information about MooseFS .INDENT 0.0 .TP -.B salt.modules.moosefs.dirinfo(path, opts=None) -Return information on a directory located on the Moose +.B salt.modules.ps.physical_memory_buffers() +Return the amount of physical memory buffers. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq moosefs.dirinfo /path/to/dir/ [\-[n][h|H]] +salt \(aq*\(aq ps.physical_memory_buffers .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.moosefs.fileinfo(path) -Return information on a file located on the Moose +.B salt.modules.ps.physical_memory_usage() +Return a dict that describes free and available physical memory. .sp -CLI Example: +CLI Examples: .sp .nf .ft C -salt \(aq*\(aq moosefs.fileinfo /path/to/dir/ +salt \(aq*\(aq ps.physical_memory_usage .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.moosefs.getgoal(path, opts=None) -Return goal(s) for a file or directory +.B salt.modules.ps.top(num_processes=5, interval=3) +Return a list of top CPU consuming processes during the interval. +num_processes = return the top N CPU consuming processes +interval = the number of seconds to sample CPU usage over +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.ps.total_physical_memory() +Return the total number of bytes of physical memory. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq moosefs.getgoal /path/to/file [\-[n][h|H]] -salt \(aq*\(aq moosefs.getgoal /path/to/dir/ [\-[n][h|H][r]] +salt \(aq*\(aq ps.total_physical_memory .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.moosefs.mounts() -Return a list of current MooseFS mounts +.B salt.modules.ps.virtual_memory_usage() +Return a dict that describes free and available memory, both physical +and virtual. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq moosefs.mounts +salt \(aq*\(aq virtual_memory_usage .ft P .fi .UNINDENT -.SS salt.modules.mount +.SS salt.modules.publish .sp -Salt module to manage unix mounts and the fstab file +Publish a command from a minion to a target .INDENT 0.0 .TP -.B salt.modules.mount.active() -List the active mounts. +.B salt.modules.publish.full_data(tgt, fun, arg=None, expr_form=\(aqglob\(aq, returner=\(aq\(aq, timeout=5) +Return the full data about the publication, this is invoked in the same +way as the publish function .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mount.active +salt system.example.com publish.full_data \(aq*\(aq cmd.run \(aqls \-la /tmp\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mount.fstab(config=\(aq/etc/fstab\(aq) -List the contents of the fstab +.B salt.modules.publish.publish(tgt, fun, arg=None, expr_form=\(aqglob\(aq, returner=\(aq\(aq, timeout=5) +Publish a command from the minion out to other minions, publications need +to be enabled on the Salt master and the minion needs to have permission +to publish the command. The Salt master will also prevent a recursive +publication loop, this means that a minion cannot command another minion +to command another minion as that would create an infinite command loop. +.sp +The arguments sent to the minion publish function are separated with +commas. This means that for a minion executing a command with multiple +args it will look like this: +.sp +.nf +.ft C +salt system.example.com publish.publish \(aq*\(aq user.add \(aqfoo,1020,1020\(aq +.ft P +.fi .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mount.fstab +salt system.example.com publish.publish \(aq*\(aq cmd.run \(aqls \-la /tmp\(aq .ft P .fi .UNINDENT +.SS salt.modules.puppet +.sp +Execute puppet routines .INDENT 0.0 .TP -.B salt.modules.mount.is_fuse_exec(cmd) -Returns true if the command passed is a fuse mountable application. +.B salt.modules.puppet.fact(name) +Run facter for a specific fact .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mount.is_fuse_exec sshfs +salt \(aq*\(aq puppet.fact kernel .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mount.mount(name, device, mkmnt=False, fstype=\(aq\(aq, opts=\(aqdefaults\(aq) -Mount a device +.B salt.modules.puppet.facts() +Run facter and return the results .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mount.mount /mnt/foo /dev/sdz1 True +salt \(aq*\(aq puppet.facts .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mount.remount(name, device, mkmnt=False, fstype=\(aq\(aq, opts=\(aqdefaults\(aq) -Attempt to remount a device, if the device is not already mounted, mount -is called +.B salt.modules.puppet.noop(tags=None) +Execute a puppet noop run and return a dict with the stderr, stdout, +return code, etc. If an argument is specified, it is treated as a +comma separated list of tags passed to puppetd \-\-test \-\-noop \-\-tags .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mount.remount /mnt/foo /dev/sdz1 True +salt \(aq*\(aq puppet.noop + +salt \(aq*\(aq puppet.noop web::server,django::base .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mount.rm_fstab(name, config=\(aq/etc/fstab\(aq) -Remove the mount point from the fstab +.B salt.modules.puppet.run(tags=None) +Execute a puppet run and return a dict with the stderr, stdout, +return code, etc. If an argument is specified, it is treated as +a comma separated list of tags passed to puppetd \-\-test \-\-tags: +\fI\%http://projects.puppetlabs.com/projects/1/wiki/Using_Tags\fP +.sp +CLI Examples: +.sp +.nf +.ft C +salt \(aq*\(aq puppet.run + +salt \(aq*\(aq puppet.run basefiles::edit,apache::server +.ft P +.fi +.UNINDENT +.SS salt.modules.pw_group +.sp +Manage groups on Linux +.INDENT 0.0 +.TP +.B salt.modules.pw_group.add(name, gid=None) +Add the specified group .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq /mnt/foo +salt \(aq*\(aq group.add foo 3456 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mount.set_fstab(name, device, fstype, opts=\(aqdefaults\(aq, dump=0, pass_num=0, config=\(aq/etc/fstab\(aq) -Verify that this mount is represented in the fstab, chage the mount point -to match the data passed, or add the mount if it is not present. +.B salt.modules.pw_group.chgid(name, gid) +Change the gid for a named group .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mount.set_fstab /mnt/foo /dev/sdz1 ext4 +salt \(aq*\(aq group.chgid foo 4376 .ft P .fi .UNINDENT -.SS salt.modules.mysql -.sp -Module to provide MySQL compatibility to salt. +.INDENT 0.0 +.TP +.B salt.modules.pw_group.delete(name) +Remove the named group .sp -In order to connect to MySQL, certain configuration is required -in /etc/salt/minion on the relevant minions. Some sample configs -might look like: +CLI Example: .sp .nf .ft C -mysql.host: \(aqlocalhost\(aq -mysql.port: 3306 -mysql.user: \(aqroot\(aq -mysql.pass: \(aq\(aq -mysql.db: \(aqmysql\(aq +salt \(aq*\(aq group.delete foo .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.pw_group.getent() +Return info on all groups .sp -You can also use a defaults file: +CLI Example: .sp .nf .ft C -mysql.default_file: \(aq/etc/mysql/debian.cnf\(aq +salt \(aq*\(aq group.getent .ft P .fi -.INDENT 0.0 -.TP -.B salt.modules.mysql.connect(**kwargs) -wrap authentication credentials here .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.db_check(name, table=None) -Repairs the full database or just a given table +.B salt.modules.pw_group.info(name) +Return information about a group .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.db_check dbname +salt \(aq*\(aq group.info foo .ft P .fi .UNINDENT +.SS salt.modules.pw_user +.sp +Manage users with the useradd command .INDENT 0.0 .TP -.B salt.modules.mysql.db_create(name) -Adds a databases to the MySQL server. +.B salt.modules.pw_user.add(name, uid=None, gid=None, groups=None, home=True, shell=None) +Add a user to the minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.db_create \(aqdbname\(aq +salt \(aq*\(aq user.add name <uid> <gid> <groups> <home> <shell> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.db_exists(name) -Checks if a database exists on the MySQL server. +.B salt.modules.pw_user.chgid(name, gid) +Change the default group of the user .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.db_exists \(aqdbname\(aq +salt \(aq*\(aq user.chgid foo 4376 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.db_list() -Return a list of databases of a MySQL server using the output -from the \fBSHOW DATABASES\fP query. +.B salt.modules.pw_user.chgroups(name, groups, append=False) +Change the groups this user belongs to, add append to append the specified +groups .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.db_list +salt \(aq*\(aq user.chgroups foo wheel,root True .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.db_optimize(name, table=None) -Optimizes the full database or just a given table +.B salt.modules.pw_user.chhome(name, home, persist=False) +Change the home directory of the user, pass true for persist to copy files +to the new home dir .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.db_optimize dbname +salt \(aq*\(aq user.chhome foo /home/users/foo True .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.db_remove(name) -Removes a databases from the MySQL server. +.B salt.modules.pw_user.chshell(name, shell) +Change the default shell of the user .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.db_remove \(aqdbname\(aq +salt \(aq*\(aq user.chshell foo /bin/zsh .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.db_repair(name, table=None) -Repairs the full database or just a given table +.B salt.modules.pw_user.chuid(name, uid) +Change the uid for a named user .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.db_repair dbname +salt \(aq*\(aq user.chuid foo 4376 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.db_tables(name) -Shows the tables in the given MySQL database (if exists) +.B salt.modules.pw_user.delete(name, remove=False, force=False) +Remove a user from the minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.db_tables \(aqdatabase\(aq +salt \(aq*\(aq user.delete name True True .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.free_slave() -Frees a slave from its master. This is a WIP, do not use. -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.mysql.grant_add(grant, database, user, host=\(aqlocalhost\(aq) -Adds a grant to the MySQL server. +.B salt.modules.pw_user.getent() +Return the list of all info for all users .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.grant_add \(aqSELECT|INSERT|UPDATE|...\(aq \(aqdatabase.*\(aq \(aqfrank\(aq \(aqlocalhost\(aq +salt \(aq*\(aq user.getent .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.grant_revoke(grant, database, user, host=\(aqlocalhost\(aq) -Removes a grant from the MySQL server. +.B salt.modules.pw_user.info(name) +Return user information .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.grant_revoke \(aqSELECT,INSERT,UPDATE\(aq \(aqdatabase.*\(aq \(aqfrank\(aq \(aqlocalhost\(aq +salt \(aq*\(aq user.info root .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.slave_lag() -Return the number of seconds that a slave SQL server is lagging behind the -master, if the host is not a slave it will return \-1. If the server is -configured to be a slave for replication but slave IO is not running then -\-2 will be returned. +.B salt.modules.pw_user.list_groups(name) +Return a list of groups the named user belongs to .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.slave_lag +salt \(aq*\(aq user.groups foo .ft P .fi .UNINDENT +.SS salt.modules.rh_service +.sp +Service support for classic Red Hat type systems. This interface uses the +service command (so it is compatible with upstart systems) and the chkconfig +command. .INDENT 0.0 .TP -.B salt.modules.mysql.status() -Return the status of a MySQL server using the output -from the \fBSHOW STATUS\fP query. +.B salt.modules.rh_service.disable(name) +Disable the named service to start at boot .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.status +salt \(aq*\(aq service.disable <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.user_chpass(user, host=\(aqlocalhost\(aq, password=None) -Change password for MySQL user +.B salt.modules.rh_service.disabled(name) +Check to see if the named service is disabled to start on boot .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.user_chpass frank localhost newpassword +salt \(aq*\(aq service.disabled <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.user_create(user, host=\(aqlocalhost\(aq, password=None) -Creates a MySQL user. +.B salt.modules.rh_service.enable(name) +Enable the named service to start at boot .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.user_create \(aqusername\(aq \(aqhostname\(aq \(aqpassword\(aq +salt \(aq*\(aq service.enable <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.user_exists(user, host=\(aqlocalhost\(aq) -Checks if a user exists on the MySQL server. +.B salt.modules.rh_service.enabled(name) +Check to see if the named service is enabled to start on boot .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.user_exists \(aqusername\(aq \(aqhostname\(aq +salt \(aq*\(aq service.enabled <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.user_grants(user, host=\(aqlocalhost\(aq) -Shows the grants for the given MySQL user (if it exists) +.B salt.modules.rh_service.get_all() +Return all installed services .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.user_grants \(aqfrank\(aq \(aqlocalhost\(aq +salt \(aq*\(aq service.get_all .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.user_info(user, host=\(aqlocalhost\(aq) -Get full info on a MySQL user +.B salt.modules.rh_service.get_disabled() +Return the disabled services .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.user_info root localhost +salt \(aq*\(aq service.get_disabled .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.user_list() -Return a list of users on a MySQL server +.B salt.modules.rh_service.get_enabled() +Return the enabled services .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.user_list +salt \(aq*\(aq service.get_enabled .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.user_remove(user, host=\(aqlocalhost\(aq) -Delete MySQL user +.B salt.modules.rh_service.restart(name) +Restart the named service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.user_remove frank localhost +salt \(aq*\(aq service.restart <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.mysql.version() -Return the version of a MySQL server using the output -from the \fBSELECT VERSION()\fP query. +.B salt.modules.rh_service.start(name) +Start the specified service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq mysql.version +salt \(aq*\(aq service.start <service name> .ft P .fi .UNINDENT -.SS salt.modules.network -.sp -Module for gathering and managing network information .INDENT 0.0 .TP -.B salt.modules.network.dig(host) -Performs a DNS lookup with dig +.B salt.modules.rh_service.status(name, sig=None) +Return the status for a service, returns a bool whether the service is +running. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq network.dig archlinux.org +salt \(aq*\(aq service.status <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.network.hwaddr(interface) -Returns the hwaddr for a given interface +.B salt.modules.rh_service.stop(name) +Stop the specified service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq network.hwaddr eth0 +salt \(aq*\(aq service.stop <service name> .ft P .fi .UNINDENT +.SS salt.modules.rvm +.sp +Manage ruby installations and gemsets with RVM, the Ruby Version Manager. +.INDENT 0.0 +.TP +.B salt.modules.rvm.do(ruby, command, runas=None) +Execute a command in an RVM controlled environment. +.INDENT 7.0 +.TP +.B ruby: +The ruby to use. +.TP +.B command: +The command to execute. +.TP +.B runas +None +The user to run rvm as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.rvm.gemset_copy(source, destination, runas=None) +Copy all gems from one gemset to another. +.INDENT 7.0 +.TP +.B source +The name of the gemset to copy, complete with ruby version. +.TP +.B destination +The destination gemset. +.TP +.B runas +None +The user to run rvm as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.rvm.gemset_create(ruby, gemset, runas=None) +Creates a gemset. +.INDENT 7.0 +.TP +.B ruby +The ruby version to create the gemset for. +.TP +.B gemset +The name of the gemset to create. +.TP +.B runas +None +The user to run rvm as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.rvm.gemset_delete(ruby, gemset, runas=None) +Deletes a gemset. +.INDENT 7.0 +.TP +.B ruby +The ruby version the gemset belongs to. +.TP +.B gemset +The gemset to delete. +.TP +.B runas +None +The user to run rvm as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.rvm.gemset_empty(ruby, gemset, runas=None) +Remove all gems from a gemset. +.INDENT 7.0 +.TP +.B ruby +The ruby version the gemset belongs to. +.TP +.B gemset +The gemset to empty. +.TP +.B runas +None +The user to run rvm as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.rvm.gemset_list(ruby=\(aqdefault\(aq, runas=None) +List all gemsets for the given ruby. +.INDENT 7.0 +.TP +.B ruby +default +The ruby version to list the gemsets for +.TP +.B runas +None +The user to run rvm as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.rvm.gemset_list_all(runas=None) +List all gemsets for all installed rubies. +.sp +Note that you must have set a default ruby before this can work. +.INDENT 7.0 +.TP +.B runas +None +The user to run rvm as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.rvm.get(version=\(aqstable\(aq, runas=None) +Update RVM. +.INDENT 7.0 +.TP +.B version +stable +Which version of RVM to install, e.g. stable or head. +.TP +.B ruby +The version of ruby to reinstall. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.rvm.install() +Install RVM system wide. +.UNINDENT .INDENT 0.0 .TP -.B salt.modules.network.interfaces() -Returns a dictionary of interfaces with various information about each -(up/down state, ip address, netmask, and hwaddr) -.sp -CLI Example: -.sp -.nf -.ft C -salt \(aq*\(aq network.interfaces -.ft P -.fi +.B salt.modules.rvm.install_ruby(ruby, runas=None) +Install a ruby implementation. +.INDENT 7.0 +.TP +.B ruby +The version of ruby to install. +.TP +.B runas +None +The user to run rvm as. +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.network.ipaddr(interface) -Returns the IP address for a given interface -.sp -CLI Example: -.sp -.nf -.ft C -salt \(aq*\(aq network.ipaddr eth0 -.ft P -.fi +.B salt.modules.rvm.is_installed() +Check if RVM is installed. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.network.isportopen(host, port) -Return status of a port -.sp -CLI Example: -.sp -.nf -.ft C -salt \(aq*\(aq network.isportopen 127.0.0.1 22 -.ft P -.fi +.B salt.modules.rvm.list(runas=None) +List all rvm installed rubies. +.INDENT 7.0 +.TP +.B runas +None +The user to run rvm as. +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.network.netmask(interface) -Returns the netmask for a given interface -.sp -CLI Example: -.sp -.nf -.ft C -salt \(aq*\(aq network.netmask eth0 -.ft P -.fi +.B salt.modules.rvm.reinstall_ruby(ruby, runas=None) +Reinstall a ruby implementation. +.INDENT 7.0 +.TP +.B ruby +The version of ruby to reinstall. +.TP +.B runas +None +The user to run rvm as. +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.network.netstat() -Return information on open ports and states -.sp -CLI Example: -.sp -.nf -.ft C -salt \(aq*\(aq network.netstat -.ft P -.fi +.B salt.modules.rvm.rubygems(ruby, version, runas=None) +Installs a specific rubygems version in the given ruby. +.INDENT 7.0 +.TP +.B ruby +The ruby to install rubygems for. +.TP +.B version +The version of rubygems to install or \(aqremove\(aq to use the version that ships with 1.9 +.TP +.B runas +None +The user to run rvm as. +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.network.ping(host) -Performs a ping to a host -.sp -CLI Example: -.sp -.nf -.ft C -salt \(aq*\(aq network.ping archlinux.org -.ft P -.fi +.B salt.modules.rvm.set_default(ruby, runas=None) +Set the default ruby. +.INDENT 7.0 +.TP +.B ruby +The version of ruby to make the default. +.TP +.B runas +None +The user to run rvm as. +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.network.traceroute(host) -Performs a traceroute to a 3rd party host +.B salt.modules.rvm.wrapper(ruby_string, wrapper_prefix, runas=None, *binaries) +Install RVM wrapper scripts. +.INDENT 7.0 +.TP +.B ruby_string +Ruby/gemset to install wrappers for. +.TP +.B wrapper_prefix +What to prepend to the name of the generated wrapper binaries. +.TP +.B runas +None +The user to run rvm as. +.TP +.B binaries +None +The names of the binaries to create wrappers for. When nothing is given, wrappers for ruby, gem, rake, irb, rdoc, ri and testrb are generated. +.UNINDENT +.UNINDENT +.SS salt.modules.saltutil +.sp +The Saltutil module is used to manage the state of the salt minion itself. It is +used to manage minion modules as well as automate updates to the salt minion +.INDENT 0.0 +.TP +.B salt.modules.saltutil.find_job(jid) +Return the data for a specific job id .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq network.traceroute archlinux.org +salt \(aq*\(aq saltutil.find_job <job id> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.network.up(interface) -Returns True if interface is up, otherwise returns False +.B salt.modules.saltutil.kill_job(jid) +Sends a termination signal (SIGTERM 15) to the named salt job\(aqs process .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq network.up eth0 +salt \(aq*\(aq saltutil.kill_job <job id> .ft P .fi .UNINDENT -.SS salt.modules.pacman -.sp -A module to wrap pacman calls, since Arch is the best -(\fI\%https://wiki.archlinux.org/index.php/Arch_is_the_best\fP) .INDENT 0.0 .TP -.B salt.modules.pacman.available_version(name) -The available version of the package in the repository +.B salt.modules.saltutil.refresh_pillar() +Queue the minion to refresh the pillar data. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.available_version <package name> +salt \(aq*\(aq saltutil.refresh_pillar .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pacman.install(name, refresh=False, **kwargs) -Install the passed package, add refresh=True to install with an \-Sy -.sp -Return a dict containing the new package names and versions: -.sp -.nf -.ft C -{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, - \(aqnew\(aq: \(aq<new\-version>\(aq]} -.ft P -.fi +.B salt.modules.saltutil.running() +Return the data on all running processes salt on the minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.install <package name> +salt \(aq*\(aq saltutil.running .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pacman.list_pkgs() -List the packages currently installed as a dict: -.sp -.nf -.ft C -{\(aq<package_name>\(aq: \(aq<version>\(aq} -.ft P -.fi +.B salt.modules.saltutil.signal_job(jid, sig) +Sends a signal to the named salt job\(aqs process .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.list_pkgs +salt \(aq*\(aq saltutil.signal_job <job id> 15 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pacman.purge(name) -Recursively remove a package and all dependencies which were installed -with it, this will call a \fBpacman \-Rsc\fP -.sp -Return a list containing the removed packages. +.B salt.modules.saltutil.sync_all(env=\(aqbase\(aq) +Sync down all of the dynamic modules from the file server for a specific +environment .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.purge <package name> +salt \(aq*\(aq saltutil.sync_all .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pacman.refresh_db() -Just run a \fBpacman \-Sy\fP, return a dict: -.sp -.nf -.ft C -{\(aq<database name>\(aq: Bool} -.ft P -.fi +.B salt.modules.saltutil.sync_grains(env=\(aqbase\(aq) +Sync the grains from the _grains directory on the salt master file +server. This function is environment aware, pass the desired environment +to grab the contents of the _grains directory, base is the default +environment. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.refresh_db +salt \(aq*\(aq saltutil.sync_grains .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pacman.remove(name) -Remove a single package with \fBpacman \-R\fP -.sp -Return a list containing the removed packages. +.B salt.modules.saltutil.sync_modules(env=\(aqbase\(aq) +Sync the modules from the _modules directory on the salt master file +server. This function is environment aware, pass the desired environment +to grab the contents of the _modules directory, base is the default +environment. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.remove <package name> +salt \(aq*\(aq saltutil.sync_modules .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pacman.upgrade() -Run a full system upgrade, a pacman \-Syu -.sp -Return a dict containing the new package names and versions: -.sp -.nf -.ft C -{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, - \(aqnew\(aq: \(aq<new\-version>\(aq]} -.ft P -.fi +.B salt.modules.saltutil.sync_renderers(env=\(aqbase\(aq) +Sync the renderers from the _renderers directory on the salt master file +server. This function is environment aware, pass the desired environment +to grab the contents of the _renderers directory, base is the default +environment. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.upgrade +salt \(aq*\(aq saltutil.sync_renderers .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pacman.version(name) -Returns a version if the package is installed, else returns an empty string +.B salt.modules.saltutil.sync_returners(env=\(aqbase\(aq) +Sync the returners from the _returners directory on the salt master file +server. This function is environment aware, pass the desired environment +to grab the contents of the _returners directory, base is the default +environment. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pkg.version <package name> +salt \(aq*\(aq saltutil.sync_returners .ft P .fi .UNINDENT -.SS salt.modules.pip -.sp -Install Python packages with pip to either the system or a virtualenv -.INDENT 0.0 -.TP -.B salt.modules.pip.freeze(env=\(aq\(aq, pip_bin=\(aq\(aq) -Return a list of installed packages either globally or in the specified -virtualenv -.INDENT 7.0 -.TP -.B env -None -The path to a virtualenv that pip should install to. This option takes -precendence over the \fBpip_bin\fP argument. -.TP -.B pip_bin -\(aqpip\(aq -The name (and optionally path) of the pip command to call. This option -will be ignored if the \fBenv\fP argument is given since it will default -to the pip that is installed in the virtualenv. This option can also be -set in the minion config file as \fBpip.pip_bin\fP. -.UNINDENT -.UNINDENT .INDENT 0.0 .TP -.B salt.modules.pip.install(env=\(aq\(aq, requirements=\(aq\(aq, pkgs=\(aq\(aq, pip_bin=\(aq\(aq) -Install packages with pip -.sp -Install packages individually or from a pip requirements file. Install -packages globally or to a virtualenv. -.INDENT 7.0 -.TP -.B env -None -The path to a virtualenv that pip should install to. This option takes -precendence over the \fBpip_bin\fP argument. -.TP -.B requirements -None -The path to a pip requirements file to install from -.TP -.B pkgs -None -A list of space\-separated packages to install -.TP -.B pip_bin -\(aqpip\(aq -The name (and optionally path) of the pip command to call. This option -will be ignored if the \fBenv\fP argument is given since it will default -to the pip that is installed in the virtualenv. This option can also be -set in the minion config file as \fBpip.pip_bin\fP. -.UNINDENT +.B salt.modules.saltutil.sync_states(env=\(aqbase\(aq) +Sync the states from the _states directory on the salt master file +server. This function is environment aware, pass the desired environment +to grab the contents of the _states directory, base is the default +environment. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pip.install /var/www/myvirtualenv.com \e - /path/to/requirements.txt +salt \(aq*\(aq saltutil.sync_states .ft P .fi .UNINDENT -.SS salt.modules.ps -.sp -A salt interface to psutil, a system and process library. -See \fI\%http://code.google.com/p/psutil\fP. .INDENT 0.0 .TP -.B salt.modules.ps.boot_time() -Return the boot time in number of seconds since the epoch began. +.B salt.modules.saltutil.term_job(jid) +Sends a termination signal (SIGTERM 15) to the named salt job\(aqs process .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ps.boot_time +salt \(aq*\(aq saltutil.term_job <job id> .ft P .fi .UNINDENT +.SS salt.modules.selinux +.sp +Execute calls on selinux .INDENT 0.0 .TP -.B salt.modules.ps.cached_physical_memory() -Return the amount cached memory. +.B salt.modules.selinux.getenforce() +Return the mode selinux is running in .sp -CLI Example: +CLE Example: .sp .nf .ft C -salt \(aq*\(aq ps.cached_physical_memory +salt \(aq*\(aq selinux.getenforce .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ps.cpu_percent(interval=0.1, per_cpu=False) -Return the percent of time the CPU is busy. -.INDENT 7.0 -.TP -.B interval -the number of seconds to sample CPU usage over -.TP -.B per_cpu -if True return an array of CPU percent busy for each CPU, otherwise -aggregate all percents into one number +.B salt.modules.selinux.setenforce(mode) +Set the enforcing mode .UNINDENT +.SS salt.modules.service +.sp +The default service module, if not otherwise specified salt will fall back +to this basic module +.INDENT 0.0 +.TP +.B salt.modules.service.reload(name) +Restart the named service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ps.cpu_percent +salt \(aq*\(aq service.reload <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ps.cpu_times(per_cpu=False) -Return the percent of time the CPU spends in each state, -e.g. user, system, idle, nice, iowait, irq, softirq. -.INDENT 7.0 -.TP -.B per_cpu -if True return an array of percents for each CPU, otherwise aggregate -all percents into one number -.UNINDENT +.B salt.modules.service.restart(name) +Restart the named service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ps.cpu_times +salt \(aq*\(aq service.restart <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ps.disk_io_counters() -Return disk I/O statisitics. +.B salt.modules.service.start(name) +Start the specified service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ps.disk_io_counters +salt \(aq*\(aq service.start <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ps.disk_partition_usage(all=False) -Return a list of disk partitions plus the mount point, filesystem and usage -statistics. +.B salt.modules.service.status(name, sig=None) +Return the status for a service, returns the PID or an empty string if the +service is running or not, pass a signature to use to find the service via +ps .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ps.disk_partition_usage +salt \(aq*\(aq service.status <service name> [service signature] .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ps.disk_partitions(all=False) -Return a list of disk partitions and their device, mount point, and -filesystem type. -.INDENT 7.0 -.TP -.B all -if set to False, only return local, physical partitions (hard disk, -USB, CD/DVD partitions). If True, return all filesystems. -.UNINDENT +.B salt.modules.service.stop(name) +Stop the specified service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ps.disk_partitions +salt \(aq*\(aq service.stop <service name> .ft P .fi .UNINDENT +.SS salt.modules.shadow +.sp +Manage the shadow file .INDENT 0.0 .TP -.B salt.modules.ps.disk_usage(path) -Given a path, return a dict listing the total available space as well as -the free space, and used space. +.B salt.modules.shadow.info(name) +Return information for the specified user .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ps.disk_usage /home +salt \(aq*\(aq shadow.info root .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ps.get_pid_list() -Return a list of process ids (PIDs) for all running processes. +.B salt.modules.shadow.set_password(name, password) +Set the password for a named user. The password must be a properly defined +hash, the password hash can be generated with this command: +\fBopenssl passwd \-1 <plaintext password>\fP .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ps.get_pid_list +salt \(aq*\(aq shadow.set_password root $1$UYCIxa628.9qXjpQCjM4a.. .ft P .fi .UNINDENT +.SS salt.modules.solr +.SS Apache Solr Salt Module +.sp +Author: Jed Glazner +Version: 0.2.1 +Modified: 12/09/2011 +.sp +This module uses http requests to talk to the apache solr request handlers +to gather information and report errors. Because of this the minion doesn\(aqt +necessarily need to reside on the actual slave. However if you want to +use the signal function the minion must reside on the physical solr host. +.sp +This module supports multi\-core and standard setups. Certain methods are +master/slave specific. Make sure you set the solr.type. If you have +questions or want a feature request please ask. +.SS Coming Features in 0.3 +.INDENT 0.0 +.IP 1. 3 +Add command for checking for replication failures on slaves +.IP 2. 3 +Improve match_index_versions since it\(aqs pointless on busy solr masters +.IP 3. 3 +Add additional local fs checks for backups to make sure they succeeded +.UNINDENT +.SS Override these in the minion config .INDENT 0.0 .TP -.B salt.modules.ps.network_io_counters() -Return network I/O statisitics. +.B solr.cores +A list of core names eg [\(aqcore1\(aq,\(aqcore2\(aq]. +An empty list indicates non\-multicore setup. +.TP +.B solr.baseurl +The root level url to access solr via http +.TP +.B solr.request_timeout +The number of seconds before timing out an http/https/ftp request. If +nothing is specified then the python global timeout setting is used. +.TP +.B solr.type +Possible values are \(aqmaster\(aq or \(aqslave\(aq +.TP +.B solr.backup_path +The path to store your backups. If you are using cores and you can specify +to append the core name to the path in the backup method. +.TP +.B solr.num_backups +For versions of solr >= 3.5. Indicates the number of backups to keep. This +option is ignored if your version is less. +.TP +.B solr.init_script +The full path to your init script with start/stop options +.TP +.B solr.dih.options +A list of options to pass to the dih. +.UNINDENT +.SS Required Options for DIH +.INDENT 0.0 +.TP +.B clean +False +Clear the index before importing +.TP +.B commit +True +Commit the documents to the index upon completion +.TP +.B optimize +True +Optimize the index after commit is complete +.TP +.B verbose +True +Get verbose output +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.solr.abort_import(handler, host=None, core_name=None, verbose=False) +MASTER ONLY +Aborts an existing import command to the specified handler. +This command can only be run if the minion is is configured with +solr.type=master +.INDENT 7.0 +.TP +.B handler +str +The name of the data import handler. +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core +str (None) +The core the handler belongs to. +.TP +.B verbose +boolean (False) +Run the command with verbose output. +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq ps.network_io_counters +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.ps.num_cpus() -Return the number of CPUs. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ps.num_cpus +salt \(aq*\(aq solr.abort_import dataimport None music {\(aqclean\(aq:True} .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ps.physical_memory_buffers() -Return the amount of physical memory buffers. +.B salt.modules.solr.backup(host=None, core_name=None, append_core_to_path=False) +Tell solr make a backup. This method can be mis\-leading since it uses the +backup api. If an error happens during the backup you are not notified. +The status: \(aqOK\(aq in the response simply means that solr received the +request successfully. +.INDENT 7.0 +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core_name +str (None) +The name of the solr core if using cores. Leave this blank if you are +not using cores or if you want to check all cores. +.TP +.B append_core_to_path +boolean (False) +If True add the name of the core to the backup path. Assumes that +minion backup path is not None. +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq ps.physical_memory_buffers +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.ps.physical_memory_usage() -Return a dict that describes free and available physical memory. .sp -CLI Examples: +CLI Example: .sp .nf .ft C -salt \(aq*\(aq ps.physical_memory_usage +salt \(aq*\(aq solr.backup music .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ps.top(num_processes=5, interval=3) -Return a list of top CPU consuming processes during the interval. -num_processes = return the top N CPU consuming processes -interval = the number of seconds to sample CPU usage over -.UNINDENT -.INDENT 0.0 +.B salt.modules.solr.core_status(host=None, core_name=None) +MULTI\-CORE HOSTS ONLY +Get the status for a given core or all cores if no core is specified +.INDENT 7.0 .TP -.B salt.modules.ps.total_physical_memory() -Return the total number of bytes of physical memory. +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core_name +str +The name of the core to reload +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq ps.total_physical_memory +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.ps.virtual_memory_usage() -Return a dict that describes free and available memory, both physical -and virtual. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virtual_memory_usage +salt \(aq*\(aq solr.core_status None music .ft P .fi .UNINDENT -.SS salt.modules.publish -.sp -Publish a command from a minion to a target .INDENT 0.0 .TP -.B salt.modules.publish.publish(tgt, fun, arg=None, expr_form=\(aqglob\(aq, returner=\(aq\(aq, timeout=5) -Publish a command from the minion out to other minions, publications need -to be enabled on the Salt master and the minion needs to have permission -to publish the command. The Salt master will also prevent a recursive -publication loop, this means that a minion cannot command another minion -to command another minion as that would create an infinite command loop. +.B salt.modules.solr.delta_import(handler, host=None, core_name=None, options={}, extra=[]) +Submits an import command to the specified handler using specified options. +This command can only be run if the minion is is configured with +solr.type=master +.INDENT 7.0 +.TP +.B handler +str +The name of the data import handler. +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core +str (None) +The core the handler belongs to. +.TP +.B options +dict (__opts__) +A list of options such as clean, optimize commit, verbose, and +pause_replication. leave blank to use __opts__ defaults. options will +be merged with __opts__ +.TP +.B extra +dict ([]) +Extra name value pairs to pass to the handler. eg ["name=value"] +.UNINDENT .sp -The arguments sent to the minion publish function are separated with -commas. This means that for a minion executing a command with multiple -args it will look like this: +Return : dict<str,obj>: .sp .nf .ft C -salt system.example.com publish.publish \(aq*\(aq user.add \(aqfoo,1020,1020\(aq +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi .sp @@ -4829,804 +9386,800 @@ CLI Example: .sp .nf .ft C -salt system.example.com publish.publish \(aq*\(aq cmd.run \(aqls \-la /tmp\(aq +salt \(aq*\(aq solr.delta_import dataimport None music {\(aqclean\(aq:True} .ft P .fi .UNINDENT -.SS salt.modules.puppet -.sp -Execute puppet routines .INDENT 0.0 .TP -.B salt.modules.puppet.fact(name) -Run facter for a specific fact -.sp -CLI Example: -.sp -.nf -.ft C -salt \(aq*\(aq puppet.fact kernel -.ft P -.fi -.UNINDENT -.INDENT 0.0 +.B salt.modules.solr.full_import(handler, host=None, core_name=None, options={}, extra=[]) +MASTER ONLY +Submits an import command to the specified handler using specified options. +This command can only be run if the minion is is configured with +solr.type=master +.INDENT 7.0 .TP -.B salt.modules.puppet.facts() -Run facter and return the results +.B handler +str +The name of the data import handler. +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core +str (None) +The core the handler belongs to. +.TP +.B options +dict (__opts__) +A list of options such as clean, optimize commit, verbose, and +pause_replication. leave blank to use __opts__ defaults. options will +be merged with __opts__ +.TP +.B extra +dict ([]) +Extra name value pairs to pass to the handler. e.g. ["name=value"] +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq puppet.facts +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.puppet.noop(tags=None) -Execute a puppet noop run and return a dict with the stderr, stdout, -return code, etc. If an argument is specified, it is treated as a -comma separated list of tags passed to puppetd \-\-test \-\-noop \-\-tags .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq puppet.noop - -salt \(aq*\(aq puppet.noop web::server,django::base +salt \(aq*\(aq solr.full_import dataimport None music {\(aqclean\(aq:True} .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.puppet.run(tags=None) -Execute a puppet run and return a dict with the stderr, stdout, -return code, etc. If an argument is specified, it is treated as -a comma separated list of tags passed to puppetd \-\-test \-\-tags: -\fI\%http://projects.puppetlabs.com/projects/1/wiki/Using_Tags\fP +.B salt.modules.solr.import_status(handler, host=None, core_name=None, verbose=False) +Submits an import command to the specified handler using specified options. +This command can only be run if the minion is is configured with +solr.type: \(aqmaster\(aq +.INDENT 7.0 +.TP +.B handler +str +The name of the data import handler. +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core +str (None) +The core the handler belongs to. +.TP +.B verbose +boolean (False) +Specifies verbose output +.UNINDENT .sp -CLI Examples: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq puppet.run - -salt \(aq*\(aq puppet.run basefiles::edit,apache::server +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.SS salt.modules.pw_group -.sp -Manage groups on Linux -.INDENT 0.0 -.TP -.B salt.modules.pw_group.add(name, gid=None) -Add the specified group .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq group.add foo 3456 +salt \(aq*\(aq solr.import_status dataimport None music False .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pw_group.chgid(name, gid) -Change the gid for a named group +.B salt.modules.solr.is_replication_enabled(host=None, core_name=None) +SLAVE CALL +Check for errors, and determine if a slave is replicating or not. +.INDENT 7.0 +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core_name +str (None) +The name of the solr core if using cores. Leave this blank if you are +not using cores or if you want to check all cores. +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq group.chgid foo 4376 +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.pw_group.delete(name) -Remove the named group .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq group.delete foo +salt \(aq*\(aq solr.is_replication_enabled music .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pw_group.getent() -Return info on all groups +.B salt.modules.solr.lucene_version(core_name=None) +Gets the lucene version that solr is using. If you are running a multi\-core +setup you should specify a core name since all the cores run under the same +servlet container, they will all have the same version. +.INDENT 7.0 +.TP +.B core_name +str (None) +The name of the solr core if using cores. Leave this blank if you are +not using cores or if you want to check all cores. +.UNINDENT .sp -CLI Example: +Return: dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq group.getent +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.pw_group.info(name) -Return information about a group .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq group.info foo +salt \(aq*\(aq solr.lucene_version .ft P .fi .UNINDENT -.SS salt.modules.pw_user -.sp -Manage users with the useradd command .INDENT 0.0 .TP -.B salt.modules.pw_user.add(name, uid=None, gid=None, groups=None, home=False, shell=\(aq/bin/false\(aq) -Add a user to the minion +.B salt.modules.solr.match_index_versions(host=None, core_name=None) +SLAVE CALL +Verifies that the master and the slave versions are in sync by +comparing the index version. If you are constantly pushing updates +the index the master and slave versions will seldom match. A solution +to this is pause indexing every so often to allow the slave to replicate +and then call this method before allowing indexing to resume. +.INDENT 7.0 +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core_name +str (None) +The name of the solr core if using cores. Leave this blank if you are +not using cores or if you want to check all cores. +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq user.add name <uid> <gid> <groups> <home> <shell> +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.pw_user.chgid(name, gid) -Change the default group of the user .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq user.chgid foo 4376 +salt \(aq*\(aq solr.match_index_versions music .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pw_user.chgroups(name, groups, append=False) -Change the groups this user belongs to, add append to append the specified -groups +.B salt.modules.solr.optimize(host=None, core_name=None) +Search queries fast, but it is a very expensive operation. The ideal +process is to run this with a master/slave configuration. Then you +can optimize the master, and push the optimized index to the slaves. +If you are running a single solr instance, or if you are going to run +this on a slave be aware than search performance will be horrible +while this command is being run. Additionally it can take a LONG time +to run and your http request may timeout. If that happens adjust your +timeout settings. +.INDENT 7.0 +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core_name +str (None) +The name of the solr core if using cores. Leave this blank if you are +not using cores or if you want to check all cores. +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq user.chgroups foo wheel,root True +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.pw_user.chhome(name, home, persist=False) -Change the home directory of the user, pass true for persist to copy files -to the new home dir .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq user.chhome foo /home/users/foo True +salt \(aq*\(aq solr.optimize music .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pw_user.chshell(name, shell) -Change the default shell of the user +.B salt.modules.solr.ping(host=None, core_name=None) +Does a health check on solr, makes sure solr can talk to the indexes. +.INDENT 7.0 +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core_name +str (None) +The name of the solr core if using cores. Leave this blank if you are +not using cores or if you want to check all cores. +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq user.chshell foo /bin/zsh +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.pw_user.chuid(name, uid) -Change the uid for a named user .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq user.chuid foo 4376 +salt \(aq*\(aq solr.ping music .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pw_user.delete(name, remove=False, force=False) -Remove a user from the minion +.B salt.modules.solr.reload_core(host=None, core_name=None) +MULTI\-CORE HOSTS ONLY +Load a new core from the same configuration as an existing registered core. +While the "new" core is initializing, the "old" one will continue to accept +requests. Once it has finished, all new request will go to the "new" core, +and the "old" core will be unloaded. +.INDENT 7.0 +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core_name +str +The name of the core to reload +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq user.delete name True True +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.pw_user.getent() -Return the list of all info for all users .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq user.getent +salt \(aq*\(aq solr.reload_core None music + +{\(aqsuccess\(aq:bool, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.pw_user.info(name) -Return user information +.B salt.modules.solr.reload_import_config(handler, host=None, core_name=None, verbose=False) +MASTER ONLY +re\-loads the handler config XML file. +This command can only be run if the minion is a \(aqmaster\(aq type +.INDENT 7.0 +.TP +.B handler +str +The name of the data import handler. +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core +str (None) +The core the handler belongs to. +.TP +.B verbose +boolean (False) +Run the command with verbose output. +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq user.info root +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.pw_user.list_groups(name) -Return a list of groups the named user belongs to .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq user.groups foo +salt \(aq*\(aq solr.reload_import_config dataimport None music {\(aqclean\(aq:True} .ft P .fi .UNINDENT -.SS salt.modules.rh_service -.sp -Top level package command wrapper, used to translate the os detected by the -grains to the correct service manager .INDENT 0.0 .TP -.B salt.modules.rh_service.disable(name) -Disable the named service to start at boot +.B salt.modules.solr.replication_details(host=None, core_name=None) +Get the full replication details. +.INDENT 7.0 +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core_name +str (None) +The name of the solr core if using cores. Leave this blank if you are +not using cores or if you want to check all cores. +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq service.disable <service name> +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.rh_service.disabled(name) -Check to see if the named service is disabled to start on boot .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.disabled <service name> +salt \(aq*\(aq solr.replication_details music .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.rh_service.enable(name) -Enable the named service to start at boot +.B salt.modules.solr.set_is_polling(polling, host=None, core_name=None) +SLAVE CALL +Prevent the slaves from polling the master for updates. +.INDENT 7.0 +.TP +.B polling +boolean +True will enable polling. False will disable it. +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core_name +str (None) +The name of the solr core if using cores. Leave this blank if you are +not using cores or if you want to check all cores. +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq service.enable <service name> +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.rh_service.enabled(name) -Check to see if the named service is enabled to start on boot .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.enabled <service name> +salt \(aq*\(aq solr.set_is_polling False .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.rh_service.get_all() -Return all installed services +.B salt.modules.solr.set_replication_enabled(status, host=None, core_name=None) +MASTER ONLY +Sets the master to ignore poll requests from the slaves. Useful when you +don\(aqt want the slaves replicating during indexing or when clearing the +index. +.INDENT 7.0 +.TP +.B status +boolean +Sets the replication status to the specified state. +.TP +.B host +str (None) +The solr host to query. __opts__[\(aqhost\(aq] is default. +.TP +.B core_name +str (None) +The name of the solr core if using cores. Leave this blank if you are +not using cores or if you want to set the status on all cores. +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq service.get_enabled +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.rh_service.get_disabled() -Return the disabled services .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.get_enabled +salt \(aq*\(aq solr.set_replication_enabled false, None, music .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.rh_service.get_enabled() -Return the enabled services +.B salt.modules.solr.signal(signal=None) +Signals Apache Solr to start, stop, or restart. Obviously this is only +going to work if the minion resides on the solr host. Additionally Solr +doesn\(aqt ship with an init script so one must be created. +.INDENT 7.0 +.TP +.B signal +str (None) +The command to pass to the apache solr init valid values are \(aqstart\(aq, +\(aqstop\(aq, and \(aqrestart\(aq +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.get_enabled +salt \(aq*\(aq solr.signal restart .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.rh_service.restart(name) -Restart the named service +.B salt.modules.solr.version(core_name=None) +Gets the solr version for the core specified. You should specify a core +here as all the cores will run under the same servlet container and so will +all have the same version. +.INDENT 7.0 +.TP +.B core_name +str (None) +The name of the solr core if using cores. Leave this blank if you are +not using cores or if you want to check all cores. +.UNINDENT .sp -CLI Example: +Return : dict<str,obj>: .sp .nf .ft C -salt \(aq*\(aq service.restart <service name> +{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.rh_service.start(name) -Start the specified service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.start <service name> +alt \(aq*\(aq solr.version .ft P .fi .UNINDENT +.SS salt.modules.ssh +.sp +Manage client ssh components .INDENT 0.0 .TP -.B salt.modules.rh_service.status(name, sig=None) -Return the status for a service, returns the PID or an empty string if the -service is running or not, pass a signature to use to find the service via -ps +.B salt.modules.ssh.auth_keys(user, config=\(aq.ssh/authorized_keys\(aq) +Return the authorized keys for the specified user .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.status <service name> [service signature] +salt \(aq*\(aq ssh.auth_keys root .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.rh_service.stop(name) -Stop the specified service +.B salt.modules.ssh.host_keys(keydir=None) +Return the minion\(aqs host keys .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.stop <service name> +salt \(aq*\(aq ssh.host_keys .ft P .fi .UNINDENT -.SS salt.modules.saltutil -.sp -The Saltutil module is used to manage the state of the salt minion itself. It is -used to manage minion modules as well as automate updates to the salt minion .INDENT 0.0 .TP -.B salt.modules.saltutil.find_job(jid) -Return the data for a specific job id +.B salt.modules.ssh.rm_auth_key(user, key, config=\(aq.ssh/authorized_keys\(aq) +Remove an authorized key from the specified user\(aqs authorized key file .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq saltutil.find_job <job id> +salt \(aq*\(aq ssh.rm_auth_key <user> <key> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.saltutil.kill_job(jid) -Sends a termination signal (SIGTERM 15) to the named salt job\(aqs process +.B salt.modules.ssh.set_auth_key(user, key, enc=\(aqssh\-rsa\(aq, comment=\(aq\(aq, options=[], config=\(aq.ssh/authorized_keys\(aq) +Add a key to the authorized_keys file .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq saltutil.kill_job <job id> +salt \(aq*\(aq ssh.set_auth_key <user> <key> dsa \(aqmy key\(aq \(aq[]\(aq .ssh/authorized_keys .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.saltutil.running() -Return the data on all running processes salt on the minion +.B salt.modules.ssh.set_auth_key_from_file(user, source, config=\(aq.ssh/authorized_keys\(aq) +Add a key to the authorized_keys file, using a file as the source. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq saltutil.running +salt \(aq*\(aq ssh.set_auth_key_from_file <user> salt://ssh_keys/<user>.id_rsa.pub .ft P .fi .UNINDENT +.SS salt.modules.state +.sp +Control the state system on the minion .INDENT 0.0 .TP -.B salt.modules.saltutil.signal_job(jid, sig) -Sends a signal to the named salt job\(aqs process +.B salt.modules.state.high(data) +Execute the compound calls stored in a single set of high data +This function is mostly intended for testing the state system .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq saltutil.signal_job <job id> 15 +salt \(aq*\(aq state.high \(aq{"vim": {"pkg": ["installed"]}}\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.saltutil.sync_all(env=\(aqbase\(aq) -Sync down all of the dynamic modules from the file server for a specific -environment +.B salt.modules.state.highstate() +Retrive the state data from the salt master for this minion and execute it .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq saltutil.sync_all +salt \(aq*\(aq state.highstate .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.saltutil.sync_grains(env=\(aqbase\(aq) -Sync the grains from the _grains directory on the salt master file -server. This function is environment aware, pass the desired environment -to grab the contents of the _grains directory, base is the default -environment. +.B salt.modules.state.low(data) +Execute a single low data call +This function is mostly intended for testing the state system .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq saltutil.sync_grains +salt \(aq*\(aq state.low \(aq{"state": "pkg", "fun": "installed", "name": "vi"}\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.saltutil.sync_modules(env=\(aqbase\(aq) -Sync the modules from the _modules directory on the salt master file -server. This function is environment aware, pass the desired environment -to grab the contents of the _modules directory, base is the default -environment. +.B salt.modules.state.show_highstate() +Retrieve the highstate data from the salt master and display it .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq saltutil.sync_modules +salt \(aq*\(aq state.show_highstate .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.saltutil.sync_renderers(env=\(aqbase\(aq) -Sync the renderers from the _renderers directory on the salt master file -server. This function is environment aware, pass the desired environment -to grab the contents of the _renderers directory, base is the default -environment. +.B salt.modules.state.show_lowstate() +List out the low data that will be applied to this minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq saltutil.sync_renderers +salt \(aq*\(aq state.show_lowstate .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.saltutil.sync_returners(env=\(aqbase\(aq) -Sync the returners from the _returners directory on the salt master file -server. This function is environment aware, pass the desired environment -to grab the contents of the _returners directory, base is the default -environment. +.B salt.modules.state.show_masterstate() +Display the data gathered from the master compiled state .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq saltutil.sync_returners +salt \(aq*\(aq state.show_masterstate .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.saltutil.sync_states(env=\(aqbase\(aq) -Sync the states from the _states directory on the salt master file -server. This function is environment aware, pass the desired environment -to grab the contents of the _states directory, base is the default -environment. +.B salt.modules.state.sls(mods, env=\(aqbase\(aq) +Execute a set list of state modules from an environment, default +environment is base .sp CLI Example: -.sp -.nf -.ft C -salt \(aq*\(aq saltutil.sync_states -.ft P -.fi +.INDENT 7.0 +.INDENT 3.5 +salt \(aq*\(aq state.sls core,edit.vim dev +.UNINDENT +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.modules.saltutil.term_job(jid) -Sends a termination signal (SIGTERM 15) to the named salt job\(aqs process +.B salt.modules.state.template(tem) +Execute the information stored in a template file on the minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq saltutil.term_job <job id> +salt \(aq*\(aq state.template \(aq<Path to template on the minion>\(aq .ft P .fi .UNINDENT -.SS salt.modules.selinux -.sp -Execute calls on selinux .INDENT 0.0 .TP -.B salt.modules.selinux.getenforce() -Return the mode selinux is running in +.B salt.modules.state.template_str(tem) +Execute the information stored in a template file on the minion .sp -CLE Example: +CLI Example: .sp .nf .ft C -salt \(aq*\(aq selinux.getenforce +salt \(aq*\(aq state.template_str \(aq<Template String>\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.selinux.setenforce(mode) -Set the enforcing mode +.B salt.modules.state.top(topfn) +Execute a specific top file instead of the default .UNINDENT -.SS salt.modules.service +.SS salt.modules.status .sp -The default service module, if not otherwise specified salt will fall back -to this basic module +Module for returning various status data about a minion. +These data can be useful for compiling into stats later. .INDENT 0.0 .TP -.B salt.modules.service.restart(name) -Restart the named service +.B salt.modules.status.all_status() +Return a composite of all status data and info for this minion. +Warning: There is a LOT here! .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.restart <service name> +salt \(aq*\(aq status.all_status .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.service.start(name) -Start the specified service +.B salt.modules.status.cpuinfo() +Return the CPU info for this minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.start <service name> +salt \(aq*\(aq status.cpuinfo .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.service.status(name, sig=None) -Return the status for a service, returns the PID or an empty string if the -service is running or not, pass a signature to use to find the service via -ps +.B salt.modules.status.cpustats() +Return the CPU stats for this minon .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.status <service name> [service signature] +salt \(aq*\(aq status.cpustats .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.service.stop(name) -Stop the specified service -.sp -CLI Example: +.B salt.modules.status.custom() +Return a custom composite of status data and info for this minon, +based on the minion config file. An example config like might be: .sp .nf .ft C -salt \(aq*\(aq service.stop <service name> +status.cpustats.custom: [ \(aqcpu\(aq, \(aqctxt\(aq, \(aqbtime\(aq, \(aqprocesses\(aq ] .ft P .fi -.UNINDENT -.SS salt.modules.shadow .sp -Manage the shadow file -.INDENT 0.0 -.TP -.B salt.modules.shadow.info(name) -Return information for the specified user +Where status refers to status.py, cpustats is the function +where we get our data, and custom is this function It is followed +by a list of keys that we want returned. +.sp +This function is meant to replace all_status(), which returns +anything and everything, which we probably don\(aqt want. +.sp +By default, nothing is returned. Warning: Depending on what you +include, there can be a LOT here! .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq shadow.info root +salt \(aq*\(aq status.custom .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.shadow.set_password(name, password) -Set the password for a named user. The password must be a properly defined -hash, the password hash can be generated with this command: -\fBopenssl passwd \-1 <plaintext password>\fP +.B salt.modules.status.diskstats() +Return the disk stats for this minion .sp CLI Example: .sp .nf -.ft C -salt \(aq*\(aq shadow.set_password root $1$UYCIxa628.9qXjpQCjM4a.. -.ft P -.fi -.UNINDENT -.SS salt.modules.solr -.SS Apache Solr Salt Module -.sp -Author: Jed Glazner -Version: 0.2.1 -Modified: 12/09/2011 -.sp -This module uses http requests to talk to the apache solr request handlers -to gather information and report errors. Because of this the minion doesn\(aqt -necessarily need to reside on the actual slave. However if you want to -use the signal function the minion must reside on the physical solr host. -.sp -This module supports multi\-core and standard setups. Certain methods are -master/slave specific. Make sure you set the solr.type. If you have -questions or want a feature request please ask. -.SS Coming Features in 0.3 -.INDENT 0.0 -.IP 1. 3 -Add command for checking for replication failures on slaves -.IP 2. 3 -Improve match_index_versions since it\(aqs pointless on busy solr masters -.IP 3. 3 -Add additional local fs checks for backups to make sure they succeeded -.UNINDENT -.SS Override these in the minion config -.INDENT 0.0 -.TP -.B solr.cores -A list of core names eg [\(aqcore1\(aq,\(aqcore2\(aq]. -An empty list indicates non\-multicore setup. -.TP -.B solr.baseurl -The root level url to access solr via http -.TP -.B solr.request_timeout -The number of seconds before timing out an http/https/ftp request. If -nothing is specified then the python global timeout setting is used. -.TP -.B solr.type -Possible values are \(aqmaster\(aq or \(aqslave\(aq -.TP -.B solr.backup_path -The path to store your backups. If you are using cores and you can specify -to append the core name to the path in the backup method. -.TP -.B solr.num_backups -For versions of solr >= 3.5. Indicates the number of backups to keep. This -option is ignored if your version is less. -.TP -.B solr.init_script -The full path to your init script with start/stop options -.TP -.B solr.dih.options -A list of options to pass to the dih. -.UNINDENT -.SS Required Options for DIH -.INDENT 0.0 -.TP -.B clean -False -Clear the index before importing -.TP -.B commit -True -Commit the documents to the index upon completion -.TP -.B optimize -True -Optimize the index after commit is complete -.TP -.B verbose -True -Get verbose output +.ft C +salt \(aq*\(aq status.diskstats +.ft P +.fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.abort_import(handler, host=None, core_name=None, verbose=False) -MASTER ONLY -Aborts an existing import command to the specified handler. -This command can only be run if the minion is is configured with -solr.type=master -.INDENT 7.0 -.TP -.B handler -str -The name of the data import handler. -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core -str (None) -The core the handler belongs to. -.TP -.B verbose -boolean (False) -Run the command with verbose output. -.UNINDENT +.B salt.modules.status.diskusage(*args) +Return the disk usage for this minion .sp -Return : dict<str,obj>: +Usage: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq status.diskusage [paths and/or filesystem types] .ft P .fi .sp @@ -5634,1873 +10187,1871 @@ CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.abort_import dataimport None music {\(aqclean\(aq:True} +salt \(aq*\(aq status.diskusage # usage for all filesystems +salt \(aq*\(aq status.diskusage / /tmp # usage for / and /tmp +salt \(aq*\(aq status.diskusage ext? # usage for ext[234] filesystems +salt \(aq*\(aq status.diskusage / ext? # usage for / and all ext filesystems .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.backup(host=None, core_name=None, append_core_to_path=False) -Tell solr make a backup. This method can be mis\-leading since it uses the -backup api. If an error happens during the backup you are not notified. -The status: \(aqOK\(aq in the response simply means that solr received the -request successfully. -.INDENT 7.0 -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core_name -str (None) -The name of the solr core if using cores. Leave this blank if you are -not using cores or if you want to check all cores. -.TP -.B append_core_to_path -boolean (False) -If True add the name of the core to the backup path. Assumes that -minion backup path is not None. -.UNINDENT +.B salt.modules.status.loadavg() +Return the load averages for this minion .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq status.loadavg .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.status.meminfo() +Return the CPU stats for this minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.backup music +salt \(aq*\(aq status.meminfo .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.core_status(host=None, core_name=None) -MULTI\-CORE HOSTS ONLY -Get the status for a given core or all cores if no core is specified -.INDENT 7.0 -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core_name -str -The name of the core to reload -.UNINDENT +.B salt.modules.status.netdev() +Return the network device stats for this minion .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq status.netdev .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.status.netstats() +Return the network stats for this minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.core_status None music +salt \(aq*\(aq status.netstats .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.delta_import(handler, host=None, core_name=None, options={}, extra=[]) -Submits an import command to the specified handler using specified options. -This command can only be run if the minion is is configured with -solr.type=master -.INDENT 7.0 -.TP -.B handler -str -The name of the data import handler. -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core -str (None) -The core the handler belongs to. -.TP -.B options -dict (__opts__) -A list of options such as clean, optimize commit, verbose, and -pause_replication. leave blank to use __opts__ defaults. options will -be merged with __opts__ -.TP -.B extra -dict ([]) -Extra name value pairs to pass to the handler. eg ["name=value"] -.UNINDENT +.B salt.modules.status.procs() +Return the process data .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq status.procs .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.status.uptime() +Return the uptime for this minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.delta_import dataimport None music {\(aqclean\(aq:True} +salt \(aq*\(aq status.uptime .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.full_import(handler, host=None, core_name=None, options={}, extra=[]) -MASTER ONLY -Submits an import command to the specified handler using specified options. -This command can only be run if the minion is is configured with -solr.type=master -.INDENT 7.0 -.TP -.B handler -str -The name of the data import handler. -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core -str (None) -The core the handler belongs to. -.TP -.B options -dict (__opts__) -A list of options such as clean, optimize commit, verbose, and -pause_replication. leave blank to use __opts__ defaults. options will -be merged with __opts__ -.TP -.B extra -dict ([]) -Extra name value pairs to pass to the handler. e.g. ["name=value"] -.UNINDENT +.B salt.modules.status.vmstats() +Return the virtual memory stats for this minion .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq status.vmstats .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.status.w() +Return a list of logged in users for this minion, using the w command .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.full_import dataimport None music {\(aqclean\(aq:True} +salt \(aq*\(aq status.w .ft P .fi .UNINDENT +.SS salt.modules.systemd +.sp +Provide the service module for systemd .INDENT 0.0 .TP -.B salt.modules.solr.import_status(handler, host=None, core_name=None, verbose=False) -Submits an import command to the specified handler using specified options. -This command can only be run if the minion is is configured with -solr.type: \(aqmaster\(aq -.INDENT 7.0 +.B salt.modules.systemd.disable(name) +Disable the named service to not start when the system boots +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq service.disable <service name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 .TP -.B handler -str -The name of the data import handler. +.B salt.modules.systemd.disabled(name) +Return if the named service is disabled to start on boot +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq service.disabled <service name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 .TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. +.B salt.modules.systemd.enable(name) +Enable the named service to start when the system boots +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq service.enable <service name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 .TP -.B core -str (None) -The core the handler belongs to. +.B salt.modules.systemd.enabled(name) +Return if the named service is enabled to start on boot +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq service.enabled <service name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 .TP -.B verbose -boolean (False) -Specifies verbose output +.B salt.modules.systemd.get_all() +Return a list of all available services +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq service.get_all +.ft P +.fi .UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.systemd.get_disabled() +Return a list of all disabled services .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq service.get_disabled .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.systemd.get_enabled() +Return a list of all enabled services .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.import_status dataimport None music False +salt \(aq*\(aq service.get_enabled .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.is_replication_enabled(host=None, core_name=None) -SLAVE CALL -Check for errors, and determine if a slave is replicating or not. -.INDENT 7.0 -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core_name -str (None) -The name of the solr core if using cores. Leave this blank if you are -not using cores or if you want to check all cores. -.UNINDENT +.B salt.modules.systemd.restart(name) +Start the specified service with systemd .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq service.start <service name> .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.systemd.start(name) +Start the specified service with systemd .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.is_replication_enabled music +salt \(aq*\(aq service.start <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.lucene_version(core_name=None) -Gets the lucene version that solr is using. If you are running a multi\-core -setup you should specify a core name since all the cores run under the same -servlet container, they will all have the same version. -.INDENT 7.0 -.TP -.B core_name -str (None) -The name of the solr core if using cores. Leave this blank if you are -not using cores or if you want to check all cores. -.UNINDENT +.B salt.modules.systemd.status(name, sig=None) +Return the status for a service via systemd, returns the PID if the service +is running or an empty string if the service is not running .sp -Return: dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq service.status <service name> .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.systemd.stop(name) +Stop the specifed service with systemd .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.lucene_version +salt \(aq*\(aq service.stop <service name> .ft P .fi .UNINDENT +.SS salt.modules.test +.sp +Module for running arbitrary tests .INDENT 0.0 .TP -.B salt.modules.solr.match_index_versions(host=None, core_name=None) -SLAVE CALL -Verifies that the master and the slave versions are in sync by -comparing the index version. If you are constantly pushing updates -the index the master and slave versions will seldom match. A solution -to this is pause indexing every so often to allow the slave to replicate -and then call this method before allowing indexing to resume. -.INDENT 7.0 -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core_name -str (None) -The name of the solr core if using cores. Leave this blank if you are -not using cores or if you want to check all cores. -.UNINDENT +.B salt.modules.test.collatz(start) +Execute the collatz conjecture from the passed starting number, +returns the sequence and the time it took to compute. Used for +performance tests. .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq test.collatz 3 .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.test.conf_test() +Return the value for test.foo in the minion configuration file, or return +the default value .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.match_index_versions music +salt \(aq*\(aq test.conf_test .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.optimize(host=None, core_name=None) -Search queries fast, but it is a very expensive operation. The ideal -process is to run this with a master/slave configuration. Then you -can optimize the master, and push the optimized index to the slaves. -If you are running a single solr instance, or if you are going to run -this on a slave be aware than search performance will be horrible -while this command is being run. Additionally it can take a LONG time -to run and your http request may timeout. If that happens adjust your -timeout settings. -.INDENT 7.0 -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core_name -str (None) -The name of the solr core if using cores. Leave this blank if you are -not using cores or if you want to check all cores. -.UNINDENT +.B salt.modules.test.cross_test(func, args=None) +Execute a minion function via the __salt__ object in the test +module, used to verify that the minion functions can be called +via the __salt__ module. .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq test.cross_test file.gid_to_group 0 .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.test.echo(text) +Return a string \- used for testing the connection .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.optimize music +salt \(aq*\(aq test.echo \(aqfoo bar baz quo qux\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.ping(host=None, core_name=None) -Does a health check on solr, makes sure solr can talk to the indexes. -.INDENT 7.0 -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core_name -str (None) -The name of the solr core if using cores. Leave this blank if you are -not using cores or if you want to check all cores. -.UNINDENT +.B salt.modules.test.fib(num) +Return a Fibonacci sequence up to the passed number, and the +timeit took to compute in seconds. Used for performance tests .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq test.fib 3 .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.test.get_opts() +Return the configuration options passed to this minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.ping music +salt \(aq*\(aq test.get_opts .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.reload_core(host=None, core_name=None) -MULTI\-CORE HOSTS ONLY -Load a new core from the same configuration as an existing registered core. -While the "new" core is initializing, the "old" one will continue to accept -requests. Once it has finished, all new request will go to the "new" core, -and the "old" core will be unloaded. -.INDENT 7.0 -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core_name -str -The name of the core to reload -.UNINDENT +.B salt.modules.test.kwarg(**kwargs) +Print out the data passed into the function \fB**kwargs\fP, this is used to +both test the publication data and cli kwarg passing, but also to display +the information available within the publication data. .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq test.kwarg .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.test.outputter(data) +Test the outputter, pass in data to return .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.reload_core None music - -{\(aqsuccess\(aq:bool, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq test.outputter foobar .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.reload_import_config(handler, host=None, core_name=None, verbose=False) -MASTER ONLY -re\-loads the handler config XML file. -This command can only be run if the minion is a \(aqmaster\(aq type -.INDENT 7.0 -.TP -.B handler -str -The name of the data import handler. -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core -str (None) -The core the handler belongs to. -.TP -.B verbose -boolean (False) -Run the command with verbose output. -.UNINDENT +.B salt.modules.test.ping() +Just used to make sure the minion is up and responding +Return True .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq test.ping .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.test.version() +Return the version of salt on the minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.reload_import_config dataimport None music {\(aqclean\(aq:True} +salt \(aq*\(aq test.version .ft P .fi .UNINDENT +.SS salt.modules.tomcat +.sp +Support for Tomcat .INDENT 0.0 .TP -.B salt.modules.solr.replication_details(host=None, core_name=None) -Get the full replication details. -.INDENT 7.0 -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core_name -str (None) -The name of the solr core if using cores. Leave this blank if you are -not using cores or if you want to check all cores. -.UNINDENT +.B salt.modules.tomcat.fullversion() +Return all server information from catalina.sh version .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq tomcat.fullversion .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.tomcat.signal(signal=None) +Signals catalina to start, stop, securestart, forcestop. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.replication_details music +salt \(aq*\(aq tomcat.signal start .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.set_is_polling(polling, host=None, core_name=None) -SLAVE CALL -Prevent the slaves from polling the master for updates. -.INDENT 7.0 -.TP -.B polling -boolean -True will enable polling. False will disable it. -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core_name -str (None) -The name of the solr core if using cores. Leave this blank if you are -not using cores or if you want to check all cores. -.UNINDENT +.B salt.modules.tomcat.version() +Return server version from catalina.sh version .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq tomcat.version .ft P .fi +.UNINDENT +.SS salt.modules.upstart +.sp +Module for the management of upstart systems. The Upstart system only supports +service starting, stopping and restarting. +.sp +DO NOT use this module on red hat systems, as red hat systems should use the +rh_service module, since red hat systems support chkconfig +.INDENT 0.0 +.TP +.B salt.modules.upstart.disable(name) +Disable the named service from starting on boot .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.set_is_polling False +salt \(aq*\(aq service.disable <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.set_replication_enabled(status, host=None, core_name=None) -MASTER ONLY -Sets the master to ignore poll requests from the slaves. Useful when you -don\(aqt want the slaves replicating during indexing or when clearing the -index. -.INDENT 7.0 -.TP -.B status -boolean -Sets the replication status to the specified state. -.TP -.B host -str (None) -The solr host to query. __opts__[\(aqhost\(aq] is default. -.TP -.B core_name -str (None) -The name of the solr core if using cores. Leave this blank if you are -not using cores or if you want to set the status on all cores. -.UNINDENT +.B salt.modules.upstart.enable(name) +Enable the named service to start at boot .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq service.enable <service name> .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.upstart.restart(name) +Restart the named service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.set_replication_enabled false, None, music +salt \(aq*\(aq service.restart <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.signal(signal=None) -Signals Apache Solr to start, stop, or restart. Obviously this is only -going to work if the minion resides on the solr host. Additionally Solr -doesn\(aqt ship with an init script so one must be created. -.INDENT 7.0 -.TP -.B signal -str (None) -The command to pass to the apache solr init valid values are \(aqstart\(aq, -\(aqstop\(aq, and \(aqrestart\(aq -.UNINDENT +.B salt.modules.upstart.start(name) +Start the specified service .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq solr.signal restart +salt \(aq*\(aq service.start <service name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.solr.version(core_name=None) -Gets the solr version for the core specified. You should specify a core -here as all the cores will run under the same servlet container and so will -all have the same version. -.INDENT 7.0 -.TP -.B core_name -str (None) -The name of the solr core if using cores. Leave this blank if you are -not using cores or if you want to check all cores. -.UNINDENT +.B salt.modules.upstart.status(name, sig=None) +Return the status for a service, returns a bool whether the service is +running. .sp -Return : dict<str,obj>: +CLI Example: .sp .nf .ft C -{\(aqsuccess\(aq:boolean, \(aqdata\(aq:dict, \(aqerrors\(aq:list, \(aqwarnings\(aq:list} +salt \(aq*\(aq service.status <service name> .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.upstart.stop(name) +Stop the specified service .sp CLI Example: .sp .nf .ft C -alt \(aq*\(aq solr.version +salt \(aq*\(aq service.stop <service name> .ft P .fi .UNINDENT -.SS salt.modules.ssh +.SS salt.modules.useradd .sp -Manage client ssh components +Manage users with the useradd command .INDENT 0.0 .TP -.B salt.modules.ssh.auth_keys(user, config=\(aq.ssh/authorized_keys\(aq) -Return the authorized keys for the specified user +.B salt.modules.useradd.add(name, uid=None, gid=None, groups=None, home=True, shell=None, fullname=None, roomnumber=None, workphone=None, homephone=None, other=None) +Add a user to the minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ssh.auth_keys root +salt \(aq*\(aq user.add name <uid> <gid> <groups> <home> <shell> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ssh.host_keys(keydir=None) -Return the minion\(aqs host keys +.B salt.modules.useradd.chfullname(name, fullname) +Change the users Full Name .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ssh.host_keys +salt \(aq*\(aq user.chfullname foo "Foo Bar" .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ssh.rm_auth_key(user, key, config=\(aq.ssh/authorized_keys\(aq) -Remove an authorized key from the specified user\(aqs authorized key file +.B salt.modules.useradd.chgid(name, gid) +Change the default group of the user .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ssh.rm_auth_key <user> <key> +salt \(aq*\(aq user.chgid foo 4376 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.ssh.set_auth_key(user, key, enc=\(aqssh\-rsa\(aq, comment=\(aq\(aq, options=[], config=\(aq.ssh/authorized_keys\(aq) -Add a key to the authorized_keys file +.B salt.modules.useradd.chgroups(name, groups, append=False) +Change the groups this user belongs to, add append to append the specified +groups .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq ssh.set_auth_key <user> <key> dsa \(aq[]\(aq .ssh/authorized_keys +salt \(aq*\(aq user.chgroups foo wheel,root True .ft P .fi .UNINDENT -.SS salt.modules.state -.sp -Control the state system on the minion .INDENT 0.0 .TP -.B salt.modules.state.high(data) -Execute the compound calls stored in a single set of high data -This function is mostly intended for testing the state system +.B salt.modules.useradd.chhome(name, home, persist=False) +Change the home directory of the user, pass true for persist to copy files +to the new home dir .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq state.high \(aq{"vim": {"pkg": ["installed"]}}\(aq +salt \(aq*\(aq user.chhome foo /home/users/foo True .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.state.highstate() -Retrive the state data from the salt master for this minion and execute it +.B salt.modules.useradd.chhomephone(name, homephone) +Change the user\(aqs Home Phone .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq state.highstate +salt \(aq*\(aq user.chhomephone foo "7735551234" .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.state.low(data) -Execute a single low data call -This function is mostly intended for testing the state system +.B salt.modules.useradd.chother(name, other) +Change the user\(aqs "Other" GECOS field .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq state.low \(aq{"state": "pkg", "fun": "installed", "name": "vi"}\(aq +salt \(aq*\(aq user.chother foo "fax=7735555678" .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.state.show_highstate() -Retrieve the highstate data from the salt master and display it +.B salt.modules.useradd.chroomnumber(name, roomnumber) +Change the user\(aqs Room Number .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq state.show_highstate +salt \(aq*\(aq user.chroomnumber foo 123 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.state.show_lowstate() -List out the low data that will be applied to this minion +.B salt.modules.useradd.chshell(name, shell) +Change the default shell of the user .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq show_lowstate +salt \(aq*\(aq user.chshell foo /bin/zsh .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.state.sls(mods, env=\(aqbase\(aq) -Execute a set list of state modules from an environment, default -environment is base +.B salt.modules.useradd.chuid(name, uid) +Change the uid for a named user .sp CLI Example: -.INDENT 7.0 -.INDENT 3.5 -salt \(aq*\(aq state.sls core,edit.vim dev -.UNINDENT -.UNINDENT +.sp +.nf +.ft C +salt \(aq*\(aq user.chuid foo 4376 +.ft P +.fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.state.template(tem) -Execute the information stored in a template file on the minion +.B salt.modules.useradd.chworkphone(name, workphone) +Change the user\(aqs Work Phone .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq state.template \(aq<Path to template on the minion>\(aq +salt \(aq*\(aq user.chworkphone foo "7735550123" .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.state.template_str(tem) -Execute the information stored in a template file on the minion +.B salt.modules.useradd.delete(name, remove=False, force=False) +Remove a user from the minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq state.template_str \(aq<Template String>\(aq +salt \(aq*\(aq user.delete name True True .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.state.top(topfn) -Execute a specific top file instead of the default -.UNINDENT -.SS salt.modules.status -.sp -Module for returning various status data about a minion. -These data can be useful for compiling into stats later. -.INDENT 0.0 -.TP -.B salt.modules.status.all_status() -Return a composite of all status data and info for this minion. -Warning: There is a LOT here! +.B salt.modules.useradd.getent() +Return the list of all info for all users .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.all_status +salt \(aq*\(aq user.getent .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.cpuinfo() -Return the CPU info for this minion +.B salt.modules.useradd.info(name) +Return user information .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.cpuinfo +salt \(aq*\(aq user.info root .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.cpustats() -Return the CPU stats for this minon +.B salt.modules.useradd.list_groups(name) +Return a list of groups the named user belongs to .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.cpustats +salt \(aq*\(aq user.groups foo .ft P .fi .UNINDENT +.SS salt.modules.virt +.sp +Work with virtual machines managed by libvirt +.sp +Required python modules: libvirt .INDENT 0.0 .TP -.B salt.modules.status.custom() -Return a custom composite of status data and info for this minon, -based on the minion config file. An example config like might be: +.B salt.modules.virt.create(vm_) +Start a defined domain +.sp +CLI Example: .sp .nf .ft C -status.cpustats.custom: [ \(aqcpu\(aq, \(aqctxt\(aq, \(aqbtime\(aq, \(aqprocesses\(aq ] +salt \(aq*\(aq virt.create <vm name> .ft P .fi -.sp -Where status refers to status.py, cpustats is the function -where we get our data, and custom is this function It is followed -by a list of keys that we want returned. -.sp -This function is meant to replace all_status(), which returns -anything and everything, which we probably don\(aqt want. -.sp -By default, nothing is returned. Warning: Depending on what you -include, there can be a LOT here! +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.virt.create_xml_path(path) +Start a defined domain .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.custom +salt \(aq*\(aq virt.create_xml_path <path to xml file on the node> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.diskstats() -Return the disk stats for this minion +.B salt.modules.virt.create_xml_str(xml) +Start a domain based on the xml passed to the function .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.diskstats +salt \(aq*\(aq virt.create_xml_str <xml in string format> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.diskusage(*args) -Return the disk usage for this minion +.B salt.modules.virt.destroy(vm_) +Hard power down the virtual machine, this is equivalent to pulling the +power .sp -Usage: +CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.diskusage [paths and/or filesystem types] +salt \(aq*\(aq virt.destroy <vm name> .ft P .fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.virt.freecpu() +Return an int representing the number of unallocated cpus on this +hypervisor .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.diskusage # usage for all filesystems -salt \(aq*\(aq status.diskusage / /tmp # usage for / and /tmp -salt \(aq*\(aq status.diskusage ext? # usage for ext[234] filesystems -salt \(aq*\(aq status.diskusage / ext? # usage for / and all ext filesystems +salt \(aq*\(aq virt.freecpu .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.loadavg() -Return the load averages for this minion +.B salt.modules.virt.freemem() +Return an int representing the amount of memory that has not been given +to virtual machines on this node .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.loadavg +salt \(aq*\(aq virt.freemem .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.meminfo() -Return the CPU stats for this minion +.B salt.modules.virt.full_info() +Return the node_info, vm_info and freemem .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.meminfo +salt \(aq*\(aq virt.full_info .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.netdev() -Return the network device stats for this minion +.B salt.modules.virt.get_disks(vm_) +Return the disks of a named vm .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.netdev +salt \(aq*\(aq virt.get_disks <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.netstats() -Return the network stats for this minion +.B salt.modules.virt.get_graphics(vm_) +Returns the information on vnc for a given vm .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.netstats +salt \(aq*\(aq virt.get_graphics <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.procs() -Return the process data +.B salt.modules.virt.get_macs(vm_) +Return a list off MAC addresses from the named vm .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.procs +salt \(aq*\(aq virt.get_macs <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.uptime() -Return the uptime for this minion +.B salt.modules.virt.get_nics(vm_) +Return info about the network interfaces of a named vm .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.uptime +salt \(aq*\(aq virt.get_nics <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.vmstats() -Return the virtual memory stats for this minion +.B salt.modules.virt.get_xml(vm_) +Returns the xml for a given vm .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.vmstats +salt \(aq*\(aq virt.get_xml <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.status.w() -Return a list of logged in users for this minion, using the w command +.B salt.modules.virt.is_hyper() +Returns a bool whether or not this nos is a hypervisor of any kind .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq status.w +salt \(aq*\(aq virt.is_hyper .ft P .fi .UNINDENT -.SS salt.modules.systemd -.sp -Provide the service module for systemd .INDENT 0.0 .TP -.B salt.modules.systemd.disable(name) -Disable the named service to not start when the system boots +.B salt.modules.virt.is_kvm_hyper() +Returns a bool whether or not this node is a KVM hypervisor .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.disable <service name> +salt \(aq*\(aq virt.is_kvm_hyper .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.systemd.disabled(name) -Return if the named service is disabled to start on boot +.B salt.modules.virt.is_xen_hyper() +Returns a bool whether or not this node is a XEN hypervisor .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.disabled <service name> +salt \(aq*\(aq virt.is_xen_hyper .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.systemd.enable(name) -Enable the named service to start when the system boots +.B salt.modules.virt.list_active_vms() +Return a list of names for active virtual machine on the minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.enable <service name> +salt \(aq*\(aq virt.list_active_vms .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.systemd.enabled(name) -Return if the named service is enabled to start on boot +.B salt.modules.virt.list_inactive_vms() +Return a list of names for inactive virtual machine on the minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.enabled <service name> +salt \(aq*\(aq virt.list_inactive_vms .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.systemd.get_all() -Return a list of all available services +.B salt.modules.virt.list_vms() +Return a list of virtual machine names on the minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.get_all +salt \(aq*\(aq virt.list_vms .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.systemd.get_disabled() -Return a list of all disabled services +.B salt.modules.virt.migrate(vm_, target) +Shared storage migration .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.get_disabled +salt \(aq*\(aq virt.migrate <vm name> <target hypervisor> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.systemd.get_enabled() -Return a list of all enabled services +.B salt.modules.virt.migrate_non_shared(vm_, target) +Attempt to execute non\-shared storage "all" migration .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.get_enabled +salt \(aq*\(aq virt.migrate_non_shared <vm name> <target hypervisor> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.systemd.restart(name) -Start the specified service with systemd +.B salt.modules.virt.migrate_non_shared_inc(vm_, target) +Attempt to execute non\-shared storage "all" migration .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.start <service name> +salt \(aq*\(aq virt.migrate_non_shared_inc <vm name> <target hypervisor> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.systemd.start(name) -Start the specified service with systemd +.B salt.modules.virt.node_info() +Return a dict with information about this node .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.start <service name> +salt \(aq*\(aq virt.node_info .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.systemd.status(name, sig=None) -Return the status for a service via systemd, returns the PID if the service -is running or an empty string if the service is not running +.B salt.modules.virt.pause(vm_) +Pause the named vm .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.status <service name> +salt \(aq*\(aq virt.pause <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.systemd.stop(name) -Stop the specifed service with systemd +.B salt.modules.virt.purge(vm_, dirs=False) +Recursively destroy and delete a virtual machine, pass True for dir\(aqs to +also delete the directories containing the virtual machine disk images \- +USE WITH EXTREME CAUTION! .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq service.stop <service name> +salt \(aq*\(aq virt.purge <vm name> .ft P .fi .UNINDENT -.SS salt.modules.test -.sp -Module for running arbitrary tests .INDENT 0.0 .TP -.B salt.modules.test.collatz(start) -Execute the collatz conjecture from the passed starting number, -returns the sequence and the time it took to compute. Used for -performance tests. +.B salt.modules.virt.resume(vm_) +Resume the named vm .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq test.collatz 3 +salt \(aq*\(aq virt.resume <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.test.conf_test() -Return the value for test.foo in the minion configuration file, or return -the default value +.B salt.modules.virt.seed_non_shared_migrate(disks, force=False) +Non shared migration requires that the disks be present on the migration +destination, pass the disks information via this function, to the +migration destination before executing the migration. .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq test.conf_test +salt \(aq*\(aq virt.seed_non_shared_migrate <disks> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.test.cross_test(func, args=[]) -Execute a minion function via the __salt__ object in the test -module, used to verify that the minion functions can be called -via the __salt__ module. +.B salt.modules.virt.set_autostart(vm_, state=\(aqon\(aq) +Set the autostart flag on a VM so that the VM will start with the host +system on reboot. +.INDENT 7.0 +.TP +.B CLI Example:: +salt "*" virt.enable_autostart <vm name> <on | off> +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.virt.shutdown(vm_) +Send a soft shutdown signal to the named vm .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq test.cross_test file.gid_to_group 0 +salt \(aq*\(aq virt.shutdown <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.test.echo(text) -Return a string \- used for testing the connection +.B salt.modules.virt.start(vm_) +Alias for the obscurely named \(aqcreate\(aq function .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq test.echo \(aqfoo bar baz quo qux\(aq +salt \(aq*\(aq virt.start <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.test.fib(num) -Return a Fibonacci sequence up to the passed number, and the -timeit took to compute in seconds. Used for performance tests +.B salt.modules.virt.undefine(vm_) +Remove a defined vm, this does not purge the virtual machine image, and +this only works if the vm is powered down .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq test.fib 3 +salt \(aq*\(aq virt.undefine <vm name> .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.test.get_opts() -Return the configuration options passed to this minion +.B salt.modules.virt.virt_type() +Returns the virtual machine type as a string .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq test.get_opts +salt \(aq*\(aq virt.virt_type .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.test.outputter(data) -Test the outputter, pass in data to return +.B salt.modules.virt.vm_info() +Return detailed information about the vms on this hyper in a dict: +.sp +.nf +.ft C +{\(aqcpu\(aq: <int>, +\(aqmaxMem\(aq: <int>, +\(aqmem\(aq: <int>, +\(aqstate\(aq: \(aq<state>\(aq, +\(aqcputime\(aq <int>} +.ft P +.fi .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq test.outputter foobar +salt \(aq*\(aq virt.vm_info .ft P .fi .UNINDENT +.SS salt.modules.virtualenv +.sp +Create virtualenv environments .INDENT 0.0 .TP -.B salt.modules.test.ping() -Just used to make sure the minion is up and responding -Return True +.B salt.modules.virtualenv.create(path, venv_bin=\(aqvirtualenv\(aq, no_site_packages=False, system_site_packages=False, distribute=False, clear=False, python=\(aq\(aq, extra_search_dir=\(aq\(aq, never_download=False, prompt=\(aq\(aq, runas=None) +Create a virtualenv +.INDENT 7.0 +.TP +.B path +The path to create the virtualenv +.TP +.B venv_bin +\(aqvirtualenv\(aq +The name (and optionally path) of the virtualenv command. This can also +be set globally in the minion config file as \fBvirtualenv.venv_bin\fP. +.TP +.B no_site_packages +False +Passthrough argument given to virtualenv +.TP +.B system_site_packages +False +Passthrough argument given to virtualenv +.TP +.B distribute +False +Passthrough argument given to virtualenv +.TP +.B clear +False +Passthrough argument given to virtualenv +.TP +.B python +(default) +Passthrough argument given to virtualenv +.TP +.B extra_search_dir +(default) +Passthrough argument given to virtualenv +.TP +.B never_download +(default) +Passthrough argument given to virtualenv +.TP +.B prompt +(default) +Passthrough argument given to virtualenv +.TP +.B runas +None +Set ownership for the virtualenv +.UNINDENT .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq test.ping +salt \(aq*\(aq pip.virtualenv /path/to/new/virtualenv .ft P .fi .UNINDENT +.SS salt.modules.win_disk +.sp +Module for gathering disk information on Windows .INDENT 0.0 .TP -.B salt.modules.test.version() -Return the version of salt on the minion +.B salt.modules.win_disk.usage() +Return usage information for volumes mounted on this minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq test.version +salt \(aq*\(aq disk.usage .ft P .fi .UNINDENT -.SS salt.modules.tomcat +.SS salt.modules.win_file .sp -Support for Tomcat +Manage information about files on the minion, set/read user, group +data +.sp +Required python modules: win32api, win32con, win32security, ntsecuritycon .INDENT 0.0 .TP -.B salt.modules.tomcat.fullversion() -Return all server information from catalina.sh version +.B salt.modules.win_file.append(path, *args) +Append text to the end of a file .sp -CLI Example: +Usage: .sp .nf .ft C -salt \(aq*\(aq tomcat.fullversion +salt \(aq*\(aq file.append /etc/motd \e + "With all thine offerings thou shalt offer salt."\e + "Salt is what makes things taste bad when it isn\(aqt in them." .ft P .fi +.sp +New in version 0.9.5. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.tomcat.signal(signal=None) -Signals catalina to start, stop, securestart, forcestop. +.B salt.modules.win_file.chgrp(path, group) +Change the group of a file .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq tomcat.signal start +salt \(aq*\(aq file.chgrp c:\etemp\etest.txt administrators .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.tomcat.version() -Return server version from catalina.sh version +.B salt.modules.win_file.chown(path, user, group) +Chown a file, pass the file the desired user and group .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq tomcat.version +salt \(aq*\(aq file.chown c:\etemp\etest.txt myusername administrators .ft P .fi .UNINDENT -.SS salt.modules.useradd -.sp -Manage users with the useradd command .INDENT 0.0 .TP -.B salt.modules.useradd.add(name, uid=None, gid=None, groups=None, home=False, shell=\(aq/bin/false\(aq) -Add a user to the minion +.B salt.modules.win_file.comment(path, regex, char=\(aq#\(aq, backup=\(aq.bak\(aq) +Comment out specified lines in a file +.INDENT 7.0 +.TP +.B path +The full path to the file to be edited +.TP +.B regex +A regular expression used to find the lines that are to be commented; +this pattern will be wrapped in parenthesis and will move any +preceding/trailing \fB^\fP or \fB$\fP characters outside the parenthesis +(e.g., the pattern \fB^foo$\fP will be rewritten as \fB^(foo)$\fP) +.TP +.B char +\fB#\fP +The character to be inserted at the beginning of a line in order to +comment it out +.TP +.B backup +\fB.bak\fP +The file will be backed up before edit with this file extension +.IP Warning +This backup will be overwritten each time \fBsed\fP / \fBcomment\fP / +\fBuncomment\fP is called. Meaning the backup will only be useful +after the first invocation. +.RE +.UNINDENT .sp -CLI Example: +Usage: .sp .nf .ft C -salt \(aq*\(aq user.add name <uid> <gid> <groups> <home> <shell> +salt \(aq*\(aq file.comment /etc/modules pcspkr .ft P .fi +.sp +New in version 0.9.5. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.useradd.chgid(name, gid) -Change the default group of the user +.B salt.modules.win_file.contains(path, text, limit=\(aq\(aq) +Return True if the file at \fBpath\fP contains \fBtext\fP .sp -CLI Example: +Usage: .sp .nf .ft C -salt \(aq*\(aq user.chgid foo 4376 +salt \(aq*\(aq file.contains /etc/crontab \(aqmymaintenance.sh\(aq .ft P .fi +.sp +New in version 0.9.5. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.useradd.chgroups(name, groups, append=False) -Change the groups this user belongs to, add append to append the specified -groups +.B salt.modules.win_file.find(path, *opts) +Approximate the Unix find(1) command and return a list of paths that +meet the specified critera. .sp -CLI Example: +The options include match criteria: .sp .nf .ft C -salt \(aq*\(aq user.chgroups foo wheel,root True +name = path\-glob # case sensitive +iname = path\-glob # case insensitive +regex = path\-regex # case sensitive +iregex = path\-regex # case insensitive +type = file\-types # match any listed type +user = users # match any listed user +group = groups # match any listed group +size = [+\-]number[size\-unit] # default unit = byte +mtime = interval # modified since date +grep = regex # search file contents .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.useradd.chhome(name, home, persist=False) -Change the home directory of the user, pass true for persist to copy files -to the new home dir .sp -CLI Example: +and/or actions: .sp .nf .ft C -salt \(aq*\(aq user.chhome foo /home/users/foo True +delete [= file\-types] # default type = \(aqf\(aq +exec = command [arg ...] # where {} is replaced by pathname +print [= print\-opts] .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.useradd.chshell(name, shell) -Change the default shell of the user .sp -CLI Example: +The default action is \(aqprint=path\(aq. +.sp +file\-glob: .sp .nf .ft C -salt \(aq*\(aq user.chshell foo /bin/zsh +* = match zero or more chars +? = match any char +[abc] = match a, b, or c +[!abc] or [^abc] = match anything except a, b, and c +[x\-y] = match chars x through y +[!x\-y] or [^x\-y] = match anything except chars x through y +{a,b,c} = match a or b or c .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.useradd.chuid(name, uid) -Change the uid for a named user .sp -CLI Example: +path\-regex: a Python re (regular expression) pattern to match pathnames +.sp +file\-types: a string of one or more of the following: .sp .nf .ft C -salt \(aq*\(aq user.chuid foo 4376 +a: all file types +b: block device +c: character device +d: directory +p: FIFO (named pipe) +f: plain file +l: symlink +s: socket .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.useradd.delete(name, remove=False, force=False) -Remove a user from the minion .sp -CLI Example: +users: a space and/or comma separated list of user names and/or uids +.sp +groups: a space and/or comma separated list of group names and/or gids +.sp +size\-unit: .sp .nf .ft C -salt \(aq*\(aq user.delete name True True +b: bytes +k: kilobytes +m: megabytes +g: gigabytes +t: terabytes .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.useradd.getent() -Return the list of all info for all users .sp -CLI Example: +interval: .sp .nf .ft C -salt \(aq*\(aq user.getent +[<num>w] [<num>[d]] [<num>h] [<num>m] [<num>s] + +where: + w: week + d: day + h: hour + m: minute + s: second .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.useradd.info(name) -Return user information .sp -CLI Example: +print\-opts: a comma and/or space separated list of one or more of the +following: .sp .nf .ft C -salt \(aq*\(aq user.info root +group: group name +md5: MD5 digest of file contents +mode: file permissions (as integer) +mtime: last modification time (as time_t) +name: file basename +path: file absolute path +size: file size in bytes +type: file type +user: user name .ft P .fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.useradd.list_groups(name) -Return a list of groups the named user belongs to .sp -CLI Example: +CLI Examples: .sp .nf .ft C -salt \(aq*\(aq user.groups foo +salt \(aq*\(aq file.find / type=f name=\e*.bak size=+10m +salt \(aq*\(aq file.find /var mtime=+30d size=+10m print=path,size,mtime +salt \(aq*\(aq file.find /var/log name=\e*.[0\-9] mtime=+30d size=+10m delete .ft P .fi .UNINDENT -.SS salt.modules.virt -.sp -Work with virtual machines managed by libvirt .INDENT 0.0 .TP -.B salt.modules.virt.create(vm_) -Start a defined domain +.B salt.modules.win_file.get_gid(path) +Return the id of the group that owns a given file .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.create <vm name> +salt \(aq*\(aq file.get_gid c:\etemp\etest.txt .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.create_xml_path(path) -Start a defined domain +.B salt.modules.win_file.get_group(path) +Return the group that owns a given file .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.create_xml_path <path to xml file on the node> +salt \(aq*\(aq file.get_group c:\etemp\etest.txt .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.create_xml_str(xml) -Start a domain based on the xml passed to the function +.B salt.modules.win_file.get_mode(path) +Return the mode of a file +.sp +Right now we\(aqre just returning 777 +because Windows\(aq doesn\(aqt have a mode +like Linux .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.create_xml_str <xml in string format> +salt \(aq*\(aq file.get_mode /etc/passwd .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.destroy(vm_) -Hard power down the virtual machine, this is equivalent to pulling the -power +.B salt.modules.win_file.get_sum(path, form=\(aqmd5\(aq) +Return the sum for the given file, default is md5, sha1, sha224, sha256, +sha384, sha512 are supported .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.destroy <vm name> +salt \(aq*\(aq file.get_sum /etc/passwd sha512 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.freecpu() -Return an int representing the number of unallocated cpus on this -hypervisor +.B salt.modules.win_file.get_uid(path) +Return the id of the user that owns a given file .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.freecpu +salt \(aq*\(aq file.get_uid c:\etemp\etest.txt .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.freemem() -Return an int representing the amount of memory that has not been given -to virtual machines on this node +.B salt.modules.win_file.get_user(path) +Return the user that owns a given file .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.freemem +salt \(aq*\(aq file.get_user c:\etemp\etest.txt .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.full_info() -Return the node_info, vm_info and freemem +.B salt.modules.win_file.gid_to_group(gid) +Convert the group id to the group name on this system .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.full_info +salt \(aq*\(aq file.gid_to_group S\-1\-5\-21\-626487655\-2533044672\-482107328\-1010 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.get_disks(vm_) -Return the disks of a named vm +.B salt.modules.win_file.group_to_gid(group) +Convert the group to the gid on this system .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.get_disks <vm name> +salt \(aq*\(aq file.group_to_gid administrators .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.get_graphics(vm_) -Returns the information on vnc for a given vm +.B salt.modules.win_file.sed(path, before, after, limit=\(aq\(aq, backup=\(aq.bak\(aq, options=\(aq\-r \-e\(aq, flags=\(aqg\(aq) +Make a simple edit to a file .sp -CLI Example: +Equivalent to: .sp .nf .ft C -salt \(aq*\(aq virt.get_graphics <vm name> +sed <backup> <options> "/<limit>/ s/<before>/<after>/<flags> <file>" .ft P .fi +.INDENT 7.0 +.TP +.B path +The full path to the file to be edited +.TP +.B before +A pattern to find in order to replace with \fBafter\fP +.TP +.B after +Text that will replace \fBbefore\fP +.TP +.B limit +\fB\(aq\(aq\fP +An initial pattern to search for before searching for \fBbefore\fP +.TP +.B backup +\fB.bak\fP +The file will be backed up before edit with this file extension; +\fBWARNING:\fP each time \fBsed\fP/\fBcomment\fP/\fBuncomment\fP is called will +overwrite this backup +.TP +.B options +\fB\-r \-e\fP +Options to pass to sed +.TP +.B flags +\fBg\fP +Flags to modify the sed search; e.g., \fBi\fP for case\-insensitve pattern +matching .UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.virt.get_xml(vm_) -Returns the xml for a given vm .sp -CLI Example: -.sp -.nf -.ft C -salt \(aq*\(aq virt.get_xml <vm name> -.ft P -.fi -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.virt.is_kvm_hyper() -Returns a bool whether or not this node is a hypervisor +Forward slashes and single quotes will be escaped automatically in the +\fBbefore\fP and \fBafter\fP patterns. .sp -CLI Example: +Usage: .sp .nf .ft C -salt \(aq*\(aq virt.is_kvm_hyper +salt \(aq*\(aq file.sed /etc/httpd/httpd.conf \(aqLogLevel warn\(aq \(aqLogLevel info\(aq .ft P .fi +.sp +New in version 0.9.5. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.list_vms() -Return a list of virtual machine names on the minion -.sp -CLI Example: +.B salt.modules.win_file.touch(name, atime=None, mtime=None) +Just like \(aqnix\(aqs "touch" command, create a file if it +doesn\(aqt exist or simply update the atime and mtime if +it already does. +.INDENT 7.0 +.TP +.B atime: +Access time in Unix epoch time +.TP +.B mtime: +Last modification in Unix epoch time +.TP +.B Usage:: +salt \(aq*\(aq file.touch /var/log/emptyfile +.UNINDENT .sp -.nf -.ft C -salt \(aq*\(aq virt.list_vms -.ft P -.fi +New in version 0.9.5. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.migrate(vm_, target) -Shared storage migration +.B salt.modules.win_file.uid_to_user(uid) +Convert a uid to a user name .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.migrate <vm name> <target hypervisor> +salt \(aq*\(aq file.uid_to_user S\-1\-5\-21\-626487655\-2533044672\-482107328\-1010 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.migrate_non_shared(vm_, target) -Attempt to execute non\-shared storage "all" migration +.B salt.modules.win_file.uncomment(path, regex, char=\(aq#\(aq, backup=\(aq.bak\(aq) +Uncomment specified commented lines in a file +.INDENT 7.0 +.TP +.B path +The full path to the file to be edited +.TP +.B regex +A regular expression used to find the lines that are to be uncommented. +This regex should not include the comment character. A leading \fB^\fP +character will be stripped for convenience (for easily switching +between comment() and uncomment()). +.TP +.B char +\fB#\fP +The character to remove in order to uncomment a line; if a single +whitespace character follows the comment it will also be removed +.TP +.B backup +\fB.bak\fP +The file will be backed up before edit with this file extension; +\fBWARNING:\fP each time \fBsed\fP/\fBcomment\fP/\fBuncomment\fP is called will +overwrite this backup +.UNINDENT .sp -CLI Example: +Usage: .sp .nf .ft C -salt \(aq*\(aq virt.migrate_non_shared <vm name> <target hypervisor> +salt \(aq*\(aq file.uncomment /etc/hosts.deny \(aqALL: PARANOID\(aq .ft P .fi +.sp +New in version 0.9.5. .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.migrate_non_shared_inc(vm_, target) -Attempt to execute non\-shared storage "all" migration +.B salt.modules.win_file.user_to_uid(user) +Convert user name to a uid .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.migrate_non_shared_inc <vm name> <target hypervisor> +salt \(aq*\(aq file.user_to_uid myusername .ft P .fi .UNINDENT +.SS salt.modules.win_network +.sp +Module for gathering and managing network information .INDENT 0.0 .TP -.B salt.modules.virt.node_info() -Return a dict with information about this node +.B salt.modules.win_network.dig(host) +Performs a DNS lookup with dig +.sp +Note: dig must be installed on the Windows minion .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.node_info +salt \(aq*\(aq network.dig archlinux.org .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.pause(vm_) -Pause the named vm +.B salt.modules.win_network.hwaddr(interface) +Returns the hwaddr for a given interface .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.pause <vm name> +salt \(aq*\(aq network.hwaddr \(aqWireless LAN adapter Wireless Network Connection\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.purge(vm_, dirs=False) -Recursively destroy and delete a virtual machine, pass True for dir\(aqs to -also delete the directories containing the virtual machine disk images \- -USE WITH EXTREME CAUTION! +.B salt.modules.win_network.interfaces() +Returns a dictionary of interfaces with various information about each +(up/down state, ip address, netmask, and hwaddr) .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.purge <vm name> +salt \(aq*\(aq network.interfaces .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.resume(vm_) -Resume the named vm +.B salt.modules.win_network.ipaddr(interface) +Returns the IP address for a given interface .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.resume <vm name> +salt \(aq*\(aq network.ipaddr \(aqWireless LAN adapter Wireless Network Connection\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.seed_non_shared_migrate(disks, force=False) -Non shared migration requires that the disks be present on the migration -destination, pass the disks information via this function, to the -migration destination before executing the migration. +.B salt.modules.win_network.isportopen(host, port) +Return status of a port .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.seed_non_shared_migrate <disks> +salt \(aq*\(aq network.isportopen 127.0.0.1 22 .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.set_autostart(vm_, state=\(aqon\(aq) -Set the autostart flag on a VM so that the VM will start with the host -system on reboot. -.INDENT 7.0 -.TP -.B CLI Example:: -salt "*" virt.enable_autostart <vm name> <on | off> -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B salt.modules.virt.shutdown(vm_) -Send a soft shutdown signal to the named vm +.B salt.modules.win_network.netmask(interface) +Returns the netmask for a given interface .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.shutdown <vm name> +salt \(aq*\(aq network.netmask \(aqWireless LAN adapter Wireless Network Connection\(aq .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.undefine(vm_) -Remove a defined vm, this does not purge the virtual machine image, and -this only works if the vm is powered down +.B salt.modules.win_network.netstat() +Return information on open ports and states .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.undefine <vm name> +salt \(aq*\(aq network.netstat .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.virt_type() -Returns the virtual machine type as a string +.B salt.modules.win_network.nslookup(host) +Query DNS for information about a domain or ip address .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.virt_type +salt \(aq*\(aq network.nslookup archlinux.org .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.virt.vm_info() -Return detailed information about the vms on this hyper in a dict: -.sp -.nf -.ft C -{\(aqcpu\(aq: <int>, -\(aqmaxMem\(aq: <int>, -\(aqmem\(aq: <int>, -\(aqstate\(aq: \(aq<state>\(aq, -\(aqcputime\(aq <int>} -.ft P -.fi +.B salt.modules.win_network.ping(host) +Performs a ping to a host .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq virt.vm_info +salt \(aq*\(aq network.ping archlinux.org .ft P .fi .UNINDENT -.SS salt.modules.virtualenv -.sp -Create virtualenv environments .INDENT 0.0 .TP -.B salt.modules.virtualenv.create(path, venv_bin=\(aq\(aq, no_site_packages=False, system_site_packages=False, clear=False, python=\(aq\(aq, extra_search_dir=\(aq\(aq, never_download=False, prompt=\(aq\(aq) -Create a virtualenv -.INDENT 7.0 -.TP -.B path -The path to create the virtualenv -.TP -.B venv_bin -\(aqvirtualenv\(aq -The name (and optionally path) of the virtualenv command. This can also -be set globally in the minion config file as \fBvirtualenv.venv_bin\fP. -.TP -.B no_site_packages -False -Passthrough argument given to virtualenv -.TP -.B system_site_packages -False -Passthrough argument given to virtualenv -.TP -.B clear -False -Passthrough argument given to virtualenv -.TP -.B python -(default) -Passthrough argument given to virtualenv -.TP -.B extra_search_dir -(default) -Passthrough argument given to virtualenv -.TP -.B never_download -(default) -Passthrough argument given to virtualenv -.TP -.B prompt -(default) -Passthrough argument given to virtualenv -.UNINDENT +.B salt.modules.win_network.traceroute(host) +Performs a traceroute to a 3rd party host .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq pip.virtualenv /path/to/new/virtualenv +salt \(aq*\(aq network.traceroute archlinux.org .ft P .fi .UNINDENT -.SS salt.modules.win_disk -.sp -Module for gathering disk information on Windows .INDENT 0.0 .TP -.B salt.modules.win_disk.usage() -Return usage information for volumes mounted on this minion +.B salt.modules.win_network.up(interface) +Returns True if interface is up, otherwise returns False .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq disk.usage +salt \(aq*\(aq network.up \(aqWireless LAN adapter Wireless Network Connection\(aq .ft P .fi .UNINDENT @@ -7657,6 +12208,36 @@ salt \(aq*\(aq service.stop <service name> .ft P .fi .UNINDENT +.SS salt.modules.win_shadow +.sp +Manage the shadow file +.INDENT 0.0 +.TP +.B salt.modules.win_shadow.info(name) +Return information for the specified user +This is just returns dummy data so that salt states can work. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq shadow.info root +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.win_shadow.set_password(name, password) +Set the password for a named user. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq shadow.set_password root mysecretpassword +.ft P +.fi +.UNINDENT .SS salt.modules.win_useradd .sp Manage Windows users with the net user command @@ -7664,7 +12245,7 @@ Manage Windows users with the net user command NOTE: This currently only works with local user accounts, not domain accounts .INDENT 0.0 .TP -.B salt.modules.win_useradd.add(name, password) +.B salt.modules.win_useradd.add(name, uid=None, gid=None, groups=None, home=False, shell=None) Add a user to the minion .sp CLI Example: @@ -7690,6 +12271,20 @@ salt \(aq*\(aq user.addgroup username groupname .UNINDENT .INDENT 0.0 .TP +.B salt.modules.win_useradd.chgroups(name, groups, append=False) +Change the groups this user belongs to, add append to append the specified +groups +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq user.chgroups foo wheel,root True +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP .B salt.modules.win_useradd.chhome(name, home) Change the home directory of the user .sp @@ -7710,20 +12305,33 @@ CLI Example: .sp .nf .ft C -salt \(aq*\(aq user.chprofile foo \e\efileserver\eprofiles\efoo +salt \(aq*\(aq user.chprofile foo \e\efileserver\eprofiles\efoo +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.win_useradd.delete(name) +Remove a user from the minion +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq user.delete name .ft P .fi .UNINDENT .INDENT 0.0 .TP -.B salt.modules.win_useradd.delete(name) -Remove a user from the minion +.B salt.modules.win_useradd.getent() +Return the list of all info for all users .sp CLI Example: .sp .nf .ft C -salt \(aq*\(aq user.delete name +salt \(aq*\(aq user.getent .ft P .fi .UNINDENT @@ -7786,6 +12394,8 @@ compatibile and uses the native yum Python interface instead of the CLI interface. .sp Support for YUM +.sp +Required python modules: yum, rpm, rpmUtils .INDENT 0.0 .TP .B salt.modules.yumpkg.available_version(name) @@ -7939,6 +12549,19 @@ salt \(aq*\(aq pkg.upgrade .UNINDENT .INDENT 0.0 .TP +.B salt.modules.yumpkg.upgrade_available(name) +Check whether or not an upgrade is available for a given package +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.upgrade_available <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP .B salt.modules.yumpkg.version(name) Returns a version if the package is installed, else returns an empty string .sp @@ -8093,6 +12716,19 @@ salt \(aq*\(aq pkg.upgrade .UNINDENT .INDENT 0.0 .TP +.B salt.modules.yumpkg5.upgrade_available(name) +Check whether or not an upgrade is available for a given package +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.upgrade_available <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP .B salt.modules.yumpkg5.version(name) Returns a version if the package is installed, else returns an empty string .sp @@ -8104,63 +12740,148 @@ salt \(aq*\(aq pkg.version <package name> .ft P .fi .UNINDENT -.SH GRAINS +.SS salt.modules.zypper .sp -Salt comes with an interface to derive information about the underlying system. -This is called the grains interface, because it presents salt with grains of -information. +Package support for openSUSE via the zypper package manager +.INDENT 0.0 +.TP +.B salt.modules.zypper.available_version(name) +Return the available version of a given package .sp -The grains interface is made available to Salt modules and components so that -the right salt minion commands are automatically available on the right -systems. +CLI Example: .sp -It is important to remember that grains are bits of information loaded when -the salt minion starts, so this information is static. This means that the -information in grains is unchanging, therefore the nature of the data is -static. So grains information are things like the running kernel, or the -operating system. -.SS Grains in the Minion Config +.nf +.ft C +salt \(aq*\(aq pkg.available_version <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.zypper.install(name, refresh=False, **kwargs) +Install the passed package, add refresh=True to install with an \-Sy .sp -Grains can also be statically assigned within the minion configuration file. -Just add the option \fBgrains\fP and pass options to it: +Return a dict containing the new package names and versions: .sp .nf .ft C -grains: - roles: - \- webserver - \- memcache - deployment: datacenter4 - cabinet: 13 - cab_u: 14\-15 +{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, + \(aqnew\(aq: \(aq<new\-version>\(aq]} .ft P .fi .sp -Then statis data specific to your servers can be retrived via Salt, or used -inside of the state system for matching. It also makes targeting, in the case -of the example above, simply based on specific data about your deployment. -.SS Writing Grains +CLI Example: .sp -Grains are easy to write. The grains interface is derived by executing all of -the "public" functions found in the modules located in the grains package or -the custom grains directory. The functions in the modules of the grains must -return a python dict, where the keys in the dict are the names of the grains and -the values are the values. +.nf +.ft C +salt \(aq*\(aq pkg.install <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.zypper.list_pkgs() +List the packages currently installed as a dict: .sp -Custom grains should be placed in a \fB_grains\fP directory located under your -\fBfile_roots\fP. Before adding a grain to salt, consider what the grain -is and remember that grains need to be static data. -.SS Examples of Grains +.nf +.ft C +{\(aq<package_name>\(aq: \(aq<version>\(aq} +.ft P +.fi .sp -The core module in the grains package is where the main grains are loaded by -the salt minion and the principal example of how to write grains: +CLI Example: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/grains/core.py\fP -.SS Syncing Grains +.nf +.ft C +salt \(aq*\(aq pkg.list_pkgs +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.zypper.purge(name) +Recursively remove a package and all dependencies which were installed +with it, this will call a \fBzypper remove \-u\fP .sp -Syncing grains can be done a number of ways, they are automatically synced when -state.highstate is called, or the grains can be synced and reloaded by calling -the saltutil.sync_grains or saltutil.sync_all functions. +Return a list containing the removed packages. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.purge <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.zypper.refresh_db() +Just run a \fBzypper refresh\fP, return a dict: +.sp +.nf +.ft C +{\(aq<database name>\(aq: Bool} +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.refresh_db +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.zypper.remove(name) +Remove a single package with \fBzypper remove\fP +.sp +Return a list containing the removed packages. +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.remove <package name> +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.zypper.upgrade() +Run a full system upgrade, a zypper upgrade +.sp +Return a dict containing the new package names and versions: +.sp +.nf +.ft C +{\(aq<package>\(aq: {\(aqold\(aq: \(aq<old\-version>\(aq, + \(aqnew\(aq: \(aq<new\-version>\(aq]} +.ft P +.fi +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.upgrade +.ft P +.fi +.UNINDENT +.INDENT 0.0 +.TP +.B salt.modules.zypper.version(name) +Returns a version if the package is installed, else returns an empty string +.sp +CLI Example: +.sp +.nf +.ft C +salt \(aq*\(aq pkg.version <package name> +.ft P +.fi +.UNINDENT .SH RETURNERS .sp By default the return values of the commands sent to the salt minions are @@ -8203,7 +12924,7 @@ salt \(aq*\(aq test.ping \-\-return mongo_return,redis_return,cassandra_return .fi .sp In this scenario all three returners will be called and the data from the -test.ping command will be sent out to the three named returers. +test.ping command will be sent out to the three named returners. .SS Writing a Returner .sp A returner is a module which contains a returner function, the returner @@ -8239,7 +12960,7 @@ serializes the data as json and sets it in redis. .SS Examples .sp The collection of built\-in salt returners can be found here: -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/returners\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/returners\fP .SH FULL LIST OF BUILTIN RETURNERS .TS center; @@ -8254,7 +12975,7 @@ _ T{ \fBcassandra_return\fP T} T{ -Return data to a Cassandra ColumFamily +Return data to a Cassandra ColumnFamily T} _ T{ @@ -8281,9 +13002,9 @@ Print the return data to the terminal to verify functionality .UNINDENT .SS salt.returners.cassandra_return .sp -Return data to a Cassandra ColumFamily +Return data to a Cassandra ColumnFamily .sp -Here\(aqs an example Keyspace/ColumnFamily setup that works with this +Here\(aqs an example Keyspace / ColumnFamily setup that works with this returner: .sp .nf @@ -8296,6 +13017,8 @@ create column family returns and default_validation_class=\(aqUTF8Type\(aq; .ft P .fi +.sp +Required python modules: pycassa .INDENT 0.0 .TP .B salt.returners.cassandra_return.returner(ret) @@ -8306,6 +13029,8 @@ Return data to a Cassandra ColumnFamily Return data to a mongodb server .sp This is the default interface for returning data for the butter statd subsytem +.sp +Required python modules: pymongo .INDENT 0.0 .TP .B salt.returners.mongo_return.returner(ret) @@ -8316,6 +13041,8 @@ Return data to a mongodb server Return data to a redis server This is a VERY simple example for pushing data to a redis server and is not necessarily intended as a usable interface. +.sp +Required python modules: redis .INDENT 0.0 .TP .B salt.returners.redis_return.returner(ret) @@ -8638,7 +13365,7 @@ python\-pkgs: .fi .sp Once converted into the \fIlowstate\fP data structure the above state -declaration will be expaneded into the following three state declarations: +declaration will be expanded into the following three state declarations: .sp .nf .ft C @@ -8721,10 +13448,9 @@ of calling one\-off executions the state of a system can be easily defined and then enforced. .SS Understanding the Salt State System Components .sp -The Salt state system is comprised of a number of components, as a user, an +The Salt state system is comprised of a number of components. As a user, an understanding of the sls and renderer systems are needed. But as a developer, -an understanding of salt states, as well as understanding salt states and how -to write the states used by salt. +an understanding of salt states and how to write the states is needed as well. .SS Salt SLS System .INDENT 0.0 .TP @@ -8770,8 +13496,8 @@ users.admin states. The init.sls files are translated to be the state name of the parent directory, so the \fBsalt/init.sls\fP file translates to the salt state. .sp -The plain files are visible to the minions, as well as the state files, in -salt, everything is a file, there is not "magic translation" of files and file +The plain files are visible to the minions, as well as the state files. In +salt, everything is a file; there is no "magic translation" of files and file types. This means that a state file can be distributed to minions just like a plain text or binary file. .SS SLS Files @@ -8779,8 +13505,8 @@ plain text or binary file. The Salt state files are simple sets of data. Since the SLS files are just data they can be represented in a number of different ways. The default format is yaml generated from a jinja template. This allows for the states files to have -all the language constructs of Python, and the simplicity of yaml. State files -can then be complicated jinja templates the translate down to yaml, or just +all the language constructs of Python and the simplicity of yaml. State files +can then be complicated jinja templates that translate down to yaml, or just plain and simple yaml files! .sp The State files are constructed data structures in a simple format. The format @@ -8857,7 +13583,7 @@ salt.master state. .sp The Renderer system is a key component to the state system. SLS files are representations of Salt "high data" structures. All Salt cares about when -reading an sls file is the data structure that is produced from file. +reading an sls file is the data structure that is produced from the file. .sp This allows Salt states to be represented by multiple types of files. The Renderer system can be used to allow different formats to be used for sls @@ -8866,7 +13592,7 @@ files. The available renderers can be found in the renderers directory in the Salt source code: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/renderers\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/renderers\fP .sp By default sls files are rendered using jinja as a templating engine, and yaml as the serialization format. Since the rendering system can be extended simply @@ -8886,6 +13612,176 @@ consist of requisite declarations and order options. Salt does \fBnot\fP execute \fIstate declarations\fP in the order they appear in the source. .RE +.SS Requisite Statements +.IP Note +This document represents behavior exhibited by Salt requisites as of +version 0.9.7 of Salt. +.RE +.sp +Often when setting up states any single action will require or depend on +another action. Salt allows you to build relationships between states with +requisite statements. A requisite statement ensure that the named state is +evaluated before the state requiring it. There are two types of requisite +statements in Salt, \fBrequire\fP and \fBwatch\fP. +.sp +These requisite statements are applied to a specific state declaration: +.sp +.nf +.ft C +httpd: + pkg: + \- installed + file: + \- managed + \- name: /etc/httpd/conf/httpd.conf + \- source: salt://httpd/httpd.conf + \- require: + \- pkg: httpd +.ft P +.fi +.sp +In this example we use the \fBrequire\fP requisite to declare that the file +/etc/httpd/conf/httpd.conf should only be set up if the pkg state executes +successfully. +.sp +The requisite system works by finding the states that are required and +executing them before the state that requires them. Then the required states +can be evaluated to see if they have executed correctly. +.SS Multiple Requisites +.sp +The requisite statement is passed as a list, allowing for the easy addition of +more requisites. Both requisite types can also be separately declared: +.sp +.nf +.ft C +httpd: + pkg: + \- installed + service: + \- running + \- enable: True + \- watch: + \- file: /etc/httpd/conf/httpd.conf + \- require: + \- pkg: httpd + \- user: httpd + \- group: httpd + file: + \- managed + \- name: /etc/httpd/conf/httpd.conf + \- source: salt://httpd/httpd.conf + \- require: + \- pkg: httpd + user: + \- present + group: + \- present +.ft P +.fi +.sp +In this example the httpd service is only going to be started if the package, +user, group and file are executed successfully. +.SS The Require Requisite +.sp +The foundation of the requisite system is the \fBrequire\fP requisite. The +require requisite ensures that the required state(s) are executed before the +requiring state. So, if a state is declared that sets down a vimrc, then it +would be pertinent to make sure that the vimrc file would only be set down if +the vim package has been installed: +.sp +.nf +.ft C +vim: + pkg: + \- installed + file: + \- managed + \- source: salt://vim/vimrc + \- require: + \- pkg: vim +.ft P +.fi +.sp +In this case, the vimrc file will only be applied by Salt if and after the vim +package is installed. +.SS The Watch Requisite +.sp +The \fBwatch\fP requisite is more advanced than the \fBrequire\fP requisite. The +watch requisite executes the same logic as require (therefore if something is +watched it does not need to also be required) with the addition of executing +logic if the required states have changed in some way. +.sp +The watch requisite checks to see if the watched states have returned any +changes. If the watched state returns changes, and the watched states execute +successfully, then the watching state will execute a function that reacts to +the changes in the watched states. +.sp +Perhaps an example can better explain the behavior: +.sp +.nf +.ft C +redis: + pkg: + \- latest + file: + \- managed + \- source: salt://redis/redis.conf + \- name: /etc/redis.conf + \- require: + \- pkg: redis + service: + \- running + \- enable: True + \- watch: + \- file: /etc/redis.conf + \- pkg: redis +.ft P +.fi +.sp +In this example the redis service will only be started if the file +/etc/redis.conf is applied, and the file is only applied if the package is +installed. This is normal require behavior, but if the watched file changes, +or the watched package is installed or upgraded, then the redis service is +restarted. +.SS Watch and the Watcher Function +.sp +The watch requisite is based on the \fBwatcher\fP function, state python +modules can include a function called watcher, this function is then called +if the watch call is invoked. In the case of the service module the underlying +service is restarted. In the case of the cmd state the command is executed. +.sp +The watcher function for the service state looks like this: +.sp +.nf +.ft C +def watcher(name, sig=None): + \(aq\(aq\(aq + The service watcher, called to invoke the watch command. + + name + The name of the init or rc script used to manage the service + + sig + The string to search for when looking for the service process with ps + \(aq\(aq\(aq + if __salt__[\(aqservice.status\(aq](name, sig): + changes = {name: __salt__[\(aqservice.restart\(aq](name)} + return {\(aqname\(aq: name, + \(aqchanges\(aq: changes, + \(aqresult\(aq: True, + \(aqcomment\(aq: \(aqService restarted\(aq} + + return {\(aqname\(aq: name, + \(aqchanges\(aq: {}, + \(aqresult\(aq: True, + \(aqcomment\(aq: \(aqService {0} started\(aq.format(name)} +.ft P +.fi +.sp +The watch requisite only works if the state that is watching has a watcher +function written. If watch is set on a state that does not have a watcher +function (like pkg), then the listed states will behave only as if they were +under a \fBrequire\fP statement. .SS The Order Option .sp Before using the order option, remember that the majority of state ordering @@ -8922,6 +13818,53 @@ vim: \- order: last .ft P .fi +.SH STATE PROVIDERS +.sp +New in version 0.9.8. +.sp +Salt predetermines what modules should be mapped to what uses based on the +properties of a system. These determinations are generally made for modules +that provide things like package and service management. +.sp +Sometimes in states it may be needed for an alternative module to be used +to provide the functionality needed. For instance, an Arch Linux system may +have been set up with systemd support, so instead of using the default service +module detected for Arch Linux, the systemd module can be used: +.sp +.nf +.ft C +httpd: + service: + \- running + \- enable: True + \- provider: systemd +.ft P +.fi +.sp +In this instance the systemd module will replace the service virtual module +which is used by default on Arch Linux, and the httpd service will be set up +using systemd. +.SS Arbitrary Module Redirects +.sp +The provider statement can also be used for more powerful means, instead of +overwriting or extending the module used for the named service an arbitrary +module can be used to provide certain functionality. +.sp +.nf +.ft C +emacs: + pkg: + \- installed + \- provider: + \- pkg: yumpkg5 + \- cmd: customcmd +.ft P +.fi +.sp +In this example the default pkg module is being redirected to use the yumpkg5 +module (yum via shelling out instead of via the yum api), but is also using +a custom module to invoke commands. This could be used to dramatically change +the behavior of a given state. .SH THE TOP FILE .sp The top file is used to map what sls modules get loaded onto what minions via @@ -9105,7 +14048,10 @@ directly define the user interface. .SS Using Custom State Modules .sp Place your custom state modules inside a \fB_states\fP directory within the -\fBfile_roots\fP specified by the master config file. +\fBfile_roots\fP specified by the master config file. These custom state modules +can then be distributed in a number of ways. Custom state modules are +distributed when state.highstate is run, or via the saltutil.sync_states +function. .SS Cross Calling Modules .sp As with Execution Modules, State Modules can also make use of the \fB__salt__\fP @@ -9140,6 +14086,71 @@ sub\-dict containing the old and new versions of the package. .IP \(bu 2 \fBcomment:\fP A string containing a summary of the result. .UNINDENT +.SS Watcher Function +.sp +If the state being written should support the watch requisite then a watcher +function needs to be declared. The watcher function is called whenever the +watch requisite is invoked and should be generic to the behavior of the state +itself. +.sp +The watcher function should accept all of the options that the normal state +functions accept (as they will be passed into the watcher function). +.sp +A watcher function typically is used to execute state specific reactive +behavior, for instance, the watcher for the service module restarts the +named service and makes it useful for the watcher to make the service +react to changes in the environment. +.sp +The watcher function also needs to return the same data that a normal state +function returns. +.SS Mod_init Interface +.sp +Some states need to execute something only once to ensure that an environment +has been set up, or certain conditions global to the state behavior can be +predefined. This is the realm of the mod_init interface. +.sp +A state module can have a function called \fBmod_init\fP which executes when the +first state of this type is called. This interface was created primarily to +improve the pkg state. When packages are installed the package metadata needs +to be refreshed, but refreshing the package metadata every time a package is +installed is wasteful. The mod_init function for the pkg state sets a flag down +so that the first, and only the first, package installation attempt will refresh +the package database (the package database can of course be manually called to +refresh via the \fBrefresh\fP option in the pkg state). +.sp +The mod_init function must accept the \fBLow State Data\fP for the given +executing state as an argument. The low state data is a dict and can be seen by +executing the state.show_lowstate function. Then the mod_init function must +return a bool. If the return value is True, then the mod_init function will not +be executed again, meaning that the needed behavior has been set up. Otherwise, +if the mod_init function returns False, then the function will be called the +next time. +.sp +A good example of the mod_init function is found in the pkg state module: +.sp +.nf +.ft C +def mod_init(low): + \(aq\(aq\(aq + Refresh the package database here so that it only needs to happen once + \(aq\(aq\(aq + if low[\(aqfun\(aq] == \(aqinstalled\(aq or low[\(aqfun\(aq] == \(aqlatest\(aq: + rtag = __gen_rtag() + if not os.path.exists(rtag): + open(rtag, \(aqw+\(aq).write(\(aq\(aq) + return True + else: + return False +.ft P +.fi +.sp +The mod_init function in the pkg state accepts the low state data as \fBlow\fP +and then checks to see if the function being called is going to install +packages, if the function is not going to install packages then there is no +need to refresh the package database. Therefore if the package database is +prepared to refresh, then return True and the mod_init will not be called +the next time a pkg state is evaluated, otherwise return False and the mod_init +will be called next time a pkg state is evaluated. .SH FULL LIST OF BUILTIN STATES .TS center; @@ -9164,6 +14175,12 @@ File Management T} _ T{ +\fBgem\fP +T} T{ +Management of rubygems +T} +_ +T{ \fBgroup\fP T} T{ Group Management @@ -9206,6 +14223,12 @@ Package Management T} _ T{ +\fBrvm\fP +T} T{ +Management of ruby installations and gemsets with RVM +T} +_ +T{ \fBservice\fP T} T{ Service Management @@ -9271,9 +14294,15 @@ syslog if there is no disk space: \- unless: echo \(aqfoo\(aq > /tmp/.test .ft P .fi +.IP Warning +Please be advised that on Unix systems the shell being used by python +to run executions is /bin/sh, this requires that commands are formatted +to execute under /bin/sh. Some capabilities of newer shells such as bash, +zsh and ksh will not always be available on minions. +.RE .INDENT 0.0 .TP -.B salt.states.cmd.run(name, onlyif=None, unless=None, cwd=\(aq/root\(aq, user=None, group=None) +.B salt.states.cmd.mod_watch(name, onlyif=None, unless=None, cwd=\(aq/root\(aq, user=None, group=None, shell=\(aq/bin/sh\(aq, env=()) Run a command if certain circumstances are met .INDENT 7.0 .TP @@ -9302,8 +14331,8 @@ The group context to run the command as .UNINDENT .INDENT 0.0 .TP -.B salt.states.cmd.wait(name, onlyif=None, unless=None, cwd=\(aq/root\(aq, user=None, group=None) -Run the given command only if the watch statement calls it +.B salt.states.cmd.run(name, onlyif=None, unless=None, cwd=\(aq/root\(aq, user=None, group=None, shell=\(aq/bin/sh\(aq, env=()) +Run a command if certain circumstances are met .INDENT 7.0 .TP .B name @@ -9331,8 +14360,8 @@ The group context to run the command as .UNINDENT .INDENT 0.0 .TP -.B salt.states.cmd.watcher(name, onlyif=None, unless=None, cwd=\(aq/root\(aq, user=None, group=None) -Run a command if certain circumstances are met +.B salt.states.cmd.wait(name, onlyif=None, unless=None, cwd=\(aq/root\(aq, user=None, group=None) +Run the given command only if the watch statement calls it .INDENT 7.0 .TP .B name @@ -9511,8 +14540,26 @@ look like this: .ft P .fi .sp +If you need to enforce user and/or group ownership recursively on the +directory\(aqs contents, you can do so by adding a \fBrecurse\fP directive: +.sp +.nf +.ft C +/srv/stuff/substuf: + file: + \- directory + \- user: fred + \- group: users + \- mode: 755 + \- makedirs: True + \- recurse: + \- user + \- group +.ft P +.fi +.sp Symlinks can be easily created, the symlink function is very simple and only -takes a few arguments +takes a few arguments: .sp .nf .ft C @@ -9603,7 +14650,7 @@ New in version 0.9.5. .UNINDENT .INDENT 0.0 .TP -.B salt.states.file.directory(name, user=None, group=None, mode=None, makedirs=False, clean=False, require=None) +.B salt.states.file.directory(name, user=None, group=None, recurse=[], mode=None, makedirs=False, clean=False, require=None) Ensure that a named directory is present and has the right perms .INDENT 7.0 .TP @@ -9618,6 +14665,9 @@ running as on the minion The group ownership set for the directory, this defaults to the group salt is running as on the minion .TP +.B recurse +Enforce user/group ownership of directory recursively +.TP .B mode The permissions to set on this directory, aka 755 .TP @@ -9631,6 +14681,9 @@ file. Make sure that only files that are set up by salt and required by this function are kept. If this option is set then everything in this directory will be deleted unless it is required. +.TP +.B require +Require other resources such as packages or files .UNINDENT .UNINDENT .INDENT 0.0 @@ -9691,7 +14744,7 @@ Default context passed to the template. .UNINDENT .INDENT 0.0 .TP -.B salt.states.file.recurse(name, source, clean=False, require=None, __env__=\(aqbase\(aq) +.B salt.states.file.recurse(name, source, clean=False, require=None, user=None, group=None, dir_mode=None, file_mode=None, __env__=\(aqbase\(aq) Recurse through a subdirectory on the master and copy said subdirecory over to the specified path. .INDENT 7.0 @@ -9709,6 +14762,23 @@ the source string is salt://spam/eggs Make sure that only files that are set up by salt and required by this function are kept. If this option is set then everything in this directory will be deleted unless it is required. +.TP +.B require +Require other resources such as packages or files +.TP +.B user +The user to own the directory, this defaults to the user salt is +running as on the minion +.TP +.B group +The group ownership set for the directory, this defaults to the group +salt is running as on the minion +.TP +.B dir_mode +The permissions mode to set any directories created +.TP +.B file_mode +The permissions mode to set any files created .UNINDENT .UNINDENT .INDENT 0.0 @@ -9803,6 +14873,58 @@ Usage: .sp New in version 0.9.5. .UNINDENT +.SS salt.states.gem +.SS Management of rubygems +.sp +A state module to manage rubygems. Gems can be set up to be installed +or removed. This module will use RVM if it is installed. In that case +you can specify what ruby version and gemset to target. +.sp +.nf +.ft C +addressable: + gem: + \- installed + \- runas: rvm + \- ruby: jruby@jgemset +.ft P +.fi +.INDENT 0.0 +.TP +.B salt.states.gem.installed(name, ruby=None, runas=None) +Make sure that a gem is installed. +.INDENT 7.0 +.TP +.B name +The name of the gem to install +.TP +.B ruby +None +For RVM installations: the ruby version and gemset to target. +.TP +.B runas +None +The user to run gem as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.states.gem.removed(name, ruby=None, runas=None) +Make sure that a gem is not installed. +.INDENT 7.0 +.TP +.B name +The name of the gem to uninstall +.TP +.B ruby +None +For RVM installations: the ruby version and gemset to target. +.TP +.B runas +None +The user to run gem as. +.UNINDENT +.UNINDENT .SS salt.states.group .SS Group Management .sp @@ -10033,12 +15155,18 @@ The name of the user to remove .UNINDENT .INDENT 0.0 .TP -.B salt.states.mysql_user.present(name, host=\(aqlocalhost\(aq, password=None) +.B salt.states.mysql_user.present(name, host=\(aqlocalhost\(aq, password=None, password_hash=None) Ensure that the named user is present with the specified properties .INDENT 7.0 .TP .B name The name of the user to manage +.TP +.B password +The password +.TP +.B password_hash +The password in hashed form .UNINDENT .UNINDENT .SS salt.states.pkg @@ -10057,7 +15185,7 @@ vim: .fi .INDENT 0.0 .TP -.B salt.states.pkg.installed(name, version=None, refresh=False, repo=\(aq\(aq, skip_verify=False) +.B salt.states.pkg.installed(name, version=None, refresh=False, repo=\(aq\(aq, skip_verify=False, **kwargs) Verify that the package is installed, and only that it is installed. This state will not upgrade an existing package and only verify that it is installed @@ -10088,7 +15216,7 @@ httpd: .UNINDENT .INDENT 0.0 .TP -.B salt.states.pkg.latest(name, refresh=False, repo=\(aq\(aq, skip_verify=False) +.B salt.states.pkg.latest(name, refresh=False, repo=\(aq\(aq, skip_verify=False, **kwargs) Verify that the named package is installed and the latest available package. If the package can be updated this state function will update the package. Generally it is better for the installed function to be @@ -10135,6 +15263,154 @@ the remove function in the salt pkg module for the platform. The name of the package to be removed .UNINDENT .UNINDENT +.SS salt.states.rvm +.SS Management of ruby installations and gemsets with RVM +.sp +This module is used to install and manage ruby installations and +gemsets with RVM, the Ruby Version Manager. Different versions of ruby +can be installed and gemsets created. RVM itself will be installed +automatically if it\(aqs not present. This module will not automatically +install packages that RVM depends on or ones that are needed to build +ruby. If you want to run RVM as an unprivileged user (recommended) you +will have to create this user yourself. This is how a state +configuration could look like: +.sp +.nf +.ft C +rvm: + group: + \- present + user: + \- present + \- gid: rvm + \- home: /home/rvm + \- require: + \- group: rvm + +rvm\-deps: + pkg: + \- installed + \- names: + \- bash + \- coreutils + \- gzip + \- bzip2 + \- gawk + \- sed + \- curl + \- git\-core + \- subversion + \- sudo + +mri\-deps: + pkg: + \- installed + \- names: + \- build\-essential + \- openssl + \- libreadline6 + \- libreadline6\-dev + \- curl + \- git\-core + \- zlib1g + \- zlib1g\-dev + \- libssl\-dev + \- libyaml\-dev + \- libsqlite3\-0 + \- libsqlite3\-dev + \- sqlite3 + \- libxml2\-dev + \- libxslt1\-dev + \- autoconf + \- libc6\-dev + \- libncurses5\-dev + \- automake + \- libtool + \- bison + \- subversion + \- ruby + +jruby\-deps: + pkg: + \- installed + \- names: + \- curl + \- g++ + \- openjdk\-6\-jre\-headless + +ruby\-1.9.2: + rvm: + \- installed + \- default: True + \- runas: rvm + \- require: + \- pkg: rvm\-deps + \- pkg: mri\-deps + \- user: rvm + +jruby: + rvm: + \- installed + \- runas: rvm + \- require: + \- pkg: rvm\-deps + \- pkg: jruby\-deps + \- user: rvm + +jgemset: + rvm: + \- gemset_present + \- ruby: jruby + \- runas: rvm + \- require: + \- rvm: jruby + +mygemset: + rvm: + \- gemset_present + \- ruby: ruby\-1.9.2 + \- runas: rvm + \- require: + \- rvm: ruby\-1.9.2 +.ft P +.fi +.INDENT 0.0 +.TP +.B salt.states.rvm.gemset_present(name, ruby=\(aqdefault\(aq, runas=None) +Verify that the gemset is present. +.INDENT 7.0 +.TP +.B name +The name of the gemset. +.TP +.B ruby +default +The ruby version this gemset belongs to. +.TP +.B runas +None +The use user to run rvm as. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B salt.states.rvm.installed(name, default=False, runas=None) +Verify that the specified ruby is installed with RVM. RVM is +installed when necessary. +.INDENT 7.0 +.TP +.B name +The version of ruby to install +.TP +.B default +False +Whether to make this ruby the default. +.TP +.B runas +None +The user to run rvm as. +.UNINDENT +.UNINDENT .SS salt.states.service .SS Service Management .sp @@ -10169,7 +15445,10 @@ The string to search for when looking for the service process with ps .INDENT 0.0 .TP .B salt.states.service.disabled(name) -Verify that the service is disabled on boot +Verify that the service is disabled on boot, only use this state if you +don\(aqt want to manage the running process, remember that if you want to +disable a service to use the enable: False option for the running or dead +function. .INDENT 7.0 .TP .B name @@ -10179,7 +15458,10 @@ The name of the init or rc script used to manage the service .INDENT 0.0 .TP .B salt.states.service.enabled(name) -Verify that the service is enabled on boot +Verify that the service is enabled on boot, only use this state if you +don\(aqt want to manage the running process, remember that if you want to +enable a running service to use the enable: True option for the running +or dead function. .INDENT 7.0 .TP .B name @@ -10188,31 +15470,31 @@ The name of the init or rc script used to manage the service .UNINDENT .INDENT 0.0 .TP -.B salt.states.service.running(name, enable=None, sig=None) -Verify that the service is running +.B salt.states.service.mod_watch(name, sig=None) +The service watcher, called to invoke the watch command. .INDENT 7.0 .TP .B name The name of the init or rc script used to manage the service .TP -.B enable -Set the service to be enabled at boot time, True sets the service to -be enabled, False sets the named service to be disabled. The default -is None, which does not enable or disable anything. -.TP .B sig The string to search for when looking for the service process with ps .UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.states.service.watcher(name, sig=None) -The service watcher, called to invoke the watch command. +.B salt.states.service.running(name, enable=None, sig=None) +Verify that the service is running .INDENT 7.0 .TP .B name The name of the init or rc script used to manage the service .TP +.B enable +Set the service to be enabled at boot time, True sets the service to +be enabled, False sets the named service to be disabled. The default +is None, which does not enable or disable anything. +.TP .B sig The string to search for when looking for the service process with ps .UNINDENT @@ -10230,6 +15512,12 @@ AAAAB3NzaC1kc3MAAACBAL0sQ9fJ5bYTEyY==: \- present \- user: root \- enc: ssh\-dss + +thatch: + ssh_auth: + \- present + \- user: root + \- source: salt://ssh_keys/thatch.id_rsa.pub .ft P .fi .INDENT 0.0 @@ -10251,7 +15539,7 @@ directory, defaults to ".ssh/authorized_keys" .UNINDENT .INDENT 0.0 .TP -.B salt.states.ssh_auth.present(name, user, enc=\(aqssh\-rsa\(aq, comment=\(aq\(aq, options=[], config=\(aq.ssh/authorized_keys\(aq) +.B salt.states.ssh_auth.present(name, user, enc=\(aqssh\-rsa\(aq, comment=\(aq\(aq, source=\(aq\(aq, options=[], config=\(aq.ssh/authorized_keys\(aq) Verifies that the specified ssh key is present for the specified user .INDENT 7.0 .TP @@ -10267,6 +15555,11 @@ Defines what type of key is being used, can be ssh\-rsa or ssh\-dss .B comment The comment to be placed with the ssh public key .TP +.B source +The source file for the key(s). Can contain any number of public keys, +in standard "authorized_keys" format. If this is set, comment, enc, +and options will be ignored. +.TP .B options The options passed to the key, pass a list object .TP @@ -10315,6 +15608,7 @@ as either absent or present fred: user: \- present + \- fullname: Fred Jones \- shell: /bin/zsh \- home: /home/fred \- uid: 4000 @@ -10344,7 +15638,7 @@ option to True to remove the user even if they are logged in .UNINDENT .INDENT 0.0 .TP -.B salt.states.user.present(name, uid=None, gid=None, groups=None, home=False, password=None, shell=\(aq/bin/bash\(aq) +.B salt.states.user.present(name, uid=None, gid=None, groups=None, home=False, password=None, enforce_password=True, shell=None, fullname=None, roomnumber=None, workphone=None, homephone=None, other=None) Ensure that the named user is present with the specified properties .INDENT 7.0 .TP @@ -10367,8 +15661,37 @@ The location of the home directory to manage .B password A password hash to set for the user .TP +.B enforce_password +Set to False to keep the password from being changed if it has already +been set and the password hash differs from what is specified in the +"password" field. This option will be ignored if "password" is not +specified. +.TP .B shell -The login shell, defaults to /bin/bash +The login shell, defaults to the system default shell +.UNINDENT +.sp +User comment field (GECOS) support (currently Linux\-only): +.sp +The below values should be specified as strings to avoid ambiguities when +the values are loaded. (Especially the phone and room number fields which +are likely to contain numeric data) +.INDENT 7.0 +.TP +.B fullname +The user\(aqs full name. +.TP +.B roomnumber +The user\(aqs room number +.TP +.B workphone +The user\(aqs work phone number +.TP +.B homephone +The user\(aqs home phone number +.TP +.B other +The user\(aqs "other" GECOS field .UNINDENT .UNINDENT .SS salt.states.virtualenv @@ -10376,7 +15699,7 @@ The login shell, defaults to /bin/bash virtualenv management .INDENT 0.0 .TP -.B salt.states.virtualenv.manage(name, venv_bin=\(aq\(aq, requirements=\(aq\(aq, no_site_packages=False, system_site_packages=False, clear=False, python=\(aq\(aq, extra_search_dir=\(aq\(aq, never_download=False, prompt=\(aq\(aq, __env__=\(aqbase\(aq) +.B salt.states.virtualenv.manage(name, venv_bin=\(aqvirtualenv\(aq, requirements=\(aq\(aq, no_site_packages=False, system_site_packages=False, distribute=False, clear=False, python=\(aq\(aq, extra_search_dir=\(aq\(aq, never_download=False, prompt=\(aq\(aq, __env__=\(aqbase\(aq, runas=None) Create a virtualenv and optionally manage it with pip .INDENT 7.0 .TP @@ -10446,7 +15769,7 @@ derived from the file. The best place to find examples of renderers is in the Salt source code. The renderers included with Salt can be found here: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/renderers\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/renderers\fP .sp Here is a simple jinja + yaml example: .sp @@ -10479,31 +15802,31 @@ center; |l|l|. _ T{ -\fBjson_jinja\fP +\fBsalt.renderers.json_jinja\fP T} T{ Process json with the jinja2 templating engine T} _ T{ -\fBjson_mako\fP +\fBsalt.renderers.json_mako\fP T} T{ Process json with the Mako templating engine T} _ T{ -\fByaml_jinja\fP +\fBsalt.renderers.yaml_jinja\fP T} T{ The default rendering engine, process yaml with the jinja2 templating engine T} _ T{ -\fByaml_mako\fP +\fBsalt.renderers.yaml_mako\fP T} T{ Process yaml with the Mako templating engine T} _ T{ -\fBpy\fP +\fBsalt.renderers.py\fP T} T{ Pure python state renderer T} @@ -10592,9 +15915,9 @@ contains a function called \fBfoo\fP then the function could be called with: .sp The best examples of runners can be found in the Salt source: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/runners\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/runners\fP .sp -A simple runner that returns a well formated list of the minons that are +A simple runner that returns a well\-formatted list of the minions that are responding to salt calls would look like this: .sp .nf @@ -10679,7 +16002,7 @@ The Salt Syndic interface is a powerful tool which allows for the construction of Salt command topologies. A basic Salt setup has a Salt Master commanding a group of Salt Minions. The Syndic interface is a special passthrough minion, it is run on a master and connects to another master, then the master -that the Syndic minion is listening to can control the minions attatched to +that the Syndic minion is listening to can control the minions attached to the master running the syndic. .sp The intent for supporting many layouts is not presented with the intent of @@ -10687,7 +16010,7 @@ supposing the use of any single topology, but to allow a more flexible method of controlling many systems. .SS Configuring the Syndic .sp -Since the Syndic only needs to be attatched to a higher level master the +Since the Syndic only needs to be attached to a higher level master the configuration is very simple. On a master that is running a syndic to connect to a higher level master the syndic_master option needs to be set in the master config file. The syndic_master option contains the hostname or ip @@ -10703,7 +16026,7 @@ extra information needs to be sent with the publications, the order_masters option makes sure that the extra data is sent out. .SS Running the Syndic .sp -The Syndic is a seperate daemon that needs to be started on the master that is +The Syndic is a separate daemon that needs to be started on the master that is controlled by a higher master. Starting the Syndic daemon is the same as starting the other Salt daemons. .sp @@ -10827,7 +16150,7 @@ Salt comes with a simple file server suitable for distributing files to the salt minions. The file server is a stateless ZeroMQ server that is built into the salt master. .sp -The main intent of the Salt File server is the present files for use in the +The main intent of the Salt File server is to present files for use in the Salt state system. With this said, the Salt file server can be used for any general file transfer from the master to the minions. .SS The cp Module @@ -10906,6 +16229,104 @@ def get_file(path, dest, env=\(aqbase\(aq): return client.get_file(path, dest, False, env) .ft P .fi +.SH FILE SERVER CONFIGURATION +.sp +The Salt file server is a high performance file server written in ZeroMQ. It +manages large files quickly and with little overhead, and has been optimized +to handle small files in an extremely efficient manner. +.sp +The Salt file server is an environment aware file server. This means that +files can be allocated within many root directories and accessed by +specifying both the file path and the environment to search. The +individual environments can span across multiple directory roots +to crate overlays and to allow for files to be organized in many flexible +ways. +.SS Environments +.sp +The Salt file server defaults to the mandatory \fBbase\fP environment. This +environment \fBMUST\fP be defined and is used to download files when no +environment is specified. +.sp +Environments allow for files and sls data to be logically separated, but +environments are not isolated from each other. This allows for logical +isolation of environments by the engineer using Salt, but also allows +for information to be used in multiple environments. +.SS Directory Overlay +.sp +The \fBenvironment\fP setting is a list of directories to publish files from. +These directories are searched in order to find the specified file and the +first file found is returned. +.sp +This means that directory data is prioritized based on the order in which they +are listed. In the case of this \fBfile_roots\fP configuration: +.sp +.nf +.ft C +file_roots: + base: + \- /srv/salt/base + \- /srv/salt/failover +.ft P +.fi +.sp +If a file\(aqs uri is \fBsalt://httpd/httpd.conf\fP, it will first search for the +file at \fB/srv/salt/base/httpd/httpd.conf\fP. If the file is found there it +will be returned. If the file is not found there, then +\fB/srv/salt/failover/httpd/httpd.conf\fP will be used for the source. +.sp +This allows for directories to be overlaid and prioritized based on the order +they are defined in the configuration. +.SS Local File Server +.sp +New in version 0.9.8. +.sp +The file server can be rerouted to run from the minion. This is primarily to +enable running salt states without a salt master. To use the local file server +interface, copy the file server data to the minion and set the file_roots +option on the minion to point to the directories copied from the master. +Once the minion \fBfile_roots\fP option has been set, change the \fBfile_client\fP +option to local to make sure that the local file server interface is used. +.SH DYNAMIC MODULE DISTRIBUTION +.sp +New in version 0.9.5. +.sp +Salt python modules can be distributed automatically via the salt file server. +Under the root of any environment defined via the file_roots option on the +master server directories corresponding to the type of module can be used. +.sp +The directories are prepended with an underscore: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.IP 1. 3 +_modules +.IP 2. 3 +_grains +.IP 3. 3 +_renderers +.IP 4. 3 +_returners +.IP 5. 3 +_states +.UNINDENT +.UNINDENT +.UNINDENT +.sp +The contents of these directories need to be synced over to the minions after +python modules have been created in them. There are a number of ways to sync +the modules. +.SS Sync Via States +.sp +The minion configuration contains an option \fBautoload_dynamic_modules\fP +which defaults to True. This option makes the state system refresh all +dynamic modules when states are run. To disable this behavior set +\fBautoload_dynamic_modules\fP to False in the minion config. +.SS Sync Via the saltutil Module +.sp +The saltutil module has a number of functions that can be used to sync all +or specific dynamic modules. The saltutil module function \fBsaltutil.sync_all\fP +will sync all module types over to a minion. For more information see: +\fBsalt.modules.saltutil\fP .SH CONFIGURATION FILE EXAMPLES .INDENT 0.0 .IP \(bu 2 @@ -11032,6 +16453,19 @@ def get_file(path, dest, env=\(aqbase\(aq): # The buffer size in the file server can be adjusted here: #file_buffer_size: 1048576 +# Pillar Configurations: +# The Salt Pillar, is a system that allows for the building of global data +# that is refined based on minion. Basically, the pillar creates data that +# can be generated to be specific based on the grains of the minion. Pillar +# is laid out in the same fashion as the file server, with environments, a top +# file and sls files. The difference is that the data does not need to be +# in the highstate format, and is generally just key/value pairs. +# +#pillar_roots: +# base: +# \- /srv/pillar +# + ##### Syndic settings ##### ########################################## # The Salt syndic is used to pass commands through a master from a higher @@ -11046,7 +16480,7 @@ def get_file(path, dest, env=\(aqbase\(aq): #order_masters: False # # If this master will be running a salt syndic daemon, syndic_master tells -# this master where to recieve commands from. +# this master where to receive commands from. #syndic_master: masterofmaster ##### Peer Publish settings ##### @@ -11121,6 +16555,11 @@ def get_file(path, dest, env=\(aqbase\(aq): # group1: \(aqL@foo.domain.com,bar.domain.com,baz.domain.com and bl*.domain.com\(aq, # group2: \(aqG@os:Debian and foo.domain.com\(aq, +##### Range Cluster settings ##### +########################################## +# The range server (and optional port) that +# serves your cluster information +#range_server: range:80 .ft P .fi @@ -11154,6 +16593,11 @@ def get_file(path, dest, env=\(aqbase\(aq): # clusters. #id: +# Append a domain to a hostname in the event that it does not exist. This is +# usefule for systems where socket.getfqdn() does not actually result in a +# FQDN (for instance, Solaris). +#append_domain: + # If the the connection to the server is interrupted, the minion will # attempt to reconnect. sub_timeout allows you to control the rate # of reconnection attempts (in seconds). To disable reconnects, set @@ -11174,6 +16618,10 @@ def get_file(path, dest, env=\(aqbase\(aq): # seconds, between those reconnection attempts. #acceptance_wait_time = 10 +# When healing a dns_check is run, this is to make sure that the originally +# resolved dns has not changed, if this is something that does not happen in +# your environment then set this value to False. +#dns_check: True ##### Minion module management ##### @@ -11192,6 +16640,14 @@ def get_file(path, dest, env=\(aqbase\(aq): #states_dirs: [] #render_dirs: [] # +# A module provider can be statically overwritten or extended for the minion +# via the providers option, in this case the default module will be +# overwritten by the specified module. In this example the pkg module will +# be provided by the yumpkg5 module instead of the system default. +# +# providers: +# pkg: yumpkg5 +# # Enable Cython modules searching and loading. (Default: False) #cython_enable: False @@ -11226,11 +16682,59 @@ def get_file(path, dest, env=\(aqbase\(aq): # enabled and can be disabled by changing this value to False #clean_dynamic_modules: True # -# Normally the minion is not isolated to any single environment on the master -# when running states, but the environment can be isolated on the minion side -# by statically setting it. Remember that the recommended way to manage -# environments is to issolate via the top file. -#environment: None +# Normally the minion is not isolated to any single environment on the master +# when running states, but the environment can be isolated on the minion side +# by statically setting it. Remember that the recommended way to manage +# environments is to issolate via the top file. +#environment: None +# +# If using the local file directory, then the state top file name needs to be +# defined, by default this is top.sls. +#state_top: top.sls + +##### File Directory Settings ##### +########################################## +# The Salt Minion can redirect all file server operations to a local directory, +# this allows for the same state tree that is on the master to be used if +# coppied completely onto the minion. This is a literal copy of the settings on +# the master but used to reference a local directory on the minion. + +# Set the file client, the client defaults to looking on the master server for +# files, but can be directed to look at the local file directory setting +# defined below by setting it to local. +#file_client: remote + +# The file directory works on environments passed to the minion, each environment +# can have multiple root directories, the subdirectories in the multiple file +# roots cannot match, otherwise the downloaded files will not be able to be +# reliably ensured. A base environment is required to house the top file. +# Example: +# file_roots: +# base: +# \- /srv/salt/ +# dev: +# \- /srv/salt/dev/services +# \- /srv/salt/dev/states +# prod: +# \- /srv/salt/prod/services +# \- /srv/salt/prod/states +# +# Default: +#file_roots: +# base: +# \- /srv/salt + +# The hash_type is the hash to use when discovering the hash of a file in +# the minion directory, the default is md5, but sha1, sha224, sha256, sha384 +# and sha512 are also supported. +#hash_type: md5 + +# The Salt pillar is searched for locally if file_client is set to local. If +# this is the case, and pillar data is defined, then the pillar_roots need to +# also be configured on the minion: +#pillar_roots: +# base: +# \- /srv/pillar ###### Security settings ##### ########################################### @@ -11362,7 +16866,7 @@ ret_port: 4506 .sp Default: \fB/\fP .sp -The system root direcotry to oporate from, change this to make Salt run from +The system root directory to operate from, change this to make Salt run from an alternative root .sp .nf @@ -11413,9 +16917,9 @@ Open mode is a dangerous security feature. One problem encountered with pki authentication systems is that keys can become "mixed up" and authentication begins to fail. Open mode turns off authentication and tells the master to accept all authentication. This will clean up the pki keys received from the -minions. Open mode should not be turned on for general use, open mode should +minions. Open mode should not be turned on for general use. Open mode should only be used for a short period of time to clean up pki keys. To turn on open -mode the value passed must be \fBTrue\fP. +mode set this value to \fBTrue\fP. .sp .nf .ft C @@ -11426,7 +16930,7 @@ open_mode: False .sp Default: \fBFalse\fP .sp -Enable auto_accept, this setting will automatically accept all incoming +Enable auto_accept. This setting will automatically accept all incoming public keys from the minions .sp .nf @@ -11493,10 +16997,10 @@ Default: \fBbase: [/srv/salt]\fP Salt runs a lightweight file server written in zeromq to deliver files to minions. This file server is built into the master daemon and does not require a dedicated port. -The file server works on environments passed to the master, each environment -can have multiple root directories, the subdirectories in the multiple file +The file server works on environments passed to the master. Each environment +can have multiple root directories. The subdirectories in the multiple file roots cannot match, otherwise the downloaded files will not be able to be -reliably ensured. A base environment is required to house the top file +reliably ensured. A base environment is required to house the top file. Example: .sp .nf @@ -11545,17 +17049,16 @@ file_buffer_size: 1048576 .fi .SS Syndic Server Settings .sp -The Salt syndic is used to pass commands through a master from a higher -master. Using the syndic is simple, if this is a master that will have -syndic servers(s) below it set the "order_masters" setting to True, if this +A Salt syndic is a Salt master used to pass commands from a higher Salt master to +minions below the syndic. Using the syndic is simple. If this is a master that +will have syndic servers(s) below it, set the "order_masters" setting to True. If this is a master that will be running a syndic daemon for passthrough the "syndic_master" setting needs to be set to the location of the master server -to recieve commands from .SS \fBorder_masters\fP .sp Default: \fBFalse\fP .sp -Extra data needs to be sind with publications if the master os controlling a +Extra data needs to be sent with publications if the master is controlling a lower level master via a syndic minion. If this is the case the order_masters value must be set to True .sp @@ -11569,7 +17072,7 @@ order_masters: False Default: \fBNone\fP .sp If this master will be running a salt\-syndic to connect to a higher level -master specify the higher level master with this configuration value +master, specify the higher level master with this configuration value .sp .nf .ft C @@ -11587,7 +17090,7 @@ compartmentalization of commands based on individual minions. Default: \fB{}\fP .sp The configuration uses regular expressions to match minions and then a list -of regular expressions to match functions, the following will allow the +of regular expressions to match functions. The following will allow the minion authenticated as foo.example.com to execute functions from the test and pkg modules .sp @@ -11610,7 +17113,7 @@ peer: .ft P .fi .sp -This is not recomanded, since it would allow anyone who gets root on any +This is not recommended, since it would allow anyone who gets root on any single minion to instantly have root on all of the minions! .SS Node Groups .sp @@ -11655,7 +17158,7 @@ log_level: warning Default: \fB{}\fP .sp Logger levels can be used to tweak specific loggers logging levels. -Imagine you want to have the salt library at the \(aqwarning\(aq level, but, you +Imagine you want to have the salt library at the \(aqwarning\(aq level, but you still wish to have \(aqsalt.modules\(aq at the \(aqdebug\(aq level: .sp .nf @@ -11741,7 +17244,7 @@ id: foo.bar.com .fi .SS \fBsub_timeout\fP .sp -The minion connection to the master may be inturupted, the minion will +The minion connection to the master may be interrupted, the minion will verify the connection every so many seconds, to disable connection verification set this value to 0 .sp @@ -11810,7 +17313,7 @@ disable_modules: .sp Default: \fB[]\fP (all returners are enabled by default) .sp -If certian returners should be disabled, this is the place +If certain returners should be disabled, this is the place .sp .nf .ft C @@ -11895,7 +17398,7 @@ renderer: yaml_jinja Default: \fBFalse\fP .sp state_verbose allows for the data returned from the minion to be more -verbose. Normaly only states that fail or states that have changes are +verbose. Normally only states that fail or states that have changes are returned, but setting state_verbose to True will return all states that were checked .sp @@ -12211,6 +17714,22 @@ The timeout in seconds to wait for replies from the salt minions. .UNINDENT .INDENT 0.0 .TP +.B \-s STATIC, \-\-static=STATIC +By default as of version 0.9.8 the salt command returns data to the +console as it is received from minions, but previous releases would return +data only after all data was received. To only return the data with a hard +timeout and after all minions have returned then use the static option. +.UNINDENT +.INDENT 0.0 +.TP +.B \-b BATCH, \-\-batch\-size=BATCH +Instead of executing on all targeted minions at once, execute on a +progressive set of minions. This option takes an argument in the form of +an explicit number of minions to execute at once, or a percentage of +minions to execute on. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-version Print the version of salt that is running. .UNINDENT @@ -12230,7 +17749,18 @@ example: server1.foo.bar,server2.foo.bar,example7.quo.qux .TP .B \-G, \-\-grain The target expression matches values returned by the salt grains system on -the minions. The target expression is in the format of \(aq<grain value>:<pcre +the minions. The target expression is in the format of \(aq<grain value>:<glob +expression>\(aq; example: \(aqos:Arch*\(aq +.sp +This was changed in version 0.9.8 to accept glob expressions instead of +regular expression. To use regular expression matching with grains use +the \-\-grain\-pcre option. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-grain\-pcre +The target expression matches values returned by the salt grains system on +the minions. The target expression is in the format of \(aq<grain value>:< regular expression>\(aq; example: \(aqos:Arch.*\(aq .UNINDENT .INDENT 0.0 @@ -12239,7 +17769,7 @@ regular expression>\(aq; example: \(aqos:Arch.*\(aq Utilize many target definitions to make the call very granular. This option takes a group of targets separated by and or or. The default matcher is a glob as usual, if something other than a glob is used preface it with the -letter denoting the type, example: \(aqwebserv* and \fI\%G@os\fP:Debian or \fI\%E@db.*\fP\(aq +letter denoting the type, example: \(aqwebserv* and \fI\%G@os\fP:Debian or \fI\%E@db*\fP\(aq make sure that the compound target is encapsulated in quotes. .UNINDENT .INDENT 0.0 @@ -12255,6 +17785,16 @@ file .UNINDENT .INDENT 0.0 .TP +.B \-R, \-\-range +Instead of using shell globs to evaluate the targe use a range expression +to identify targets. Range expressions look like %cluster. +.sp +Using the Range option requires that a range server is set up and the +location of the range server is referenced in the master configuration +file. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-return Chose an alternative returner to call on the minion, if an alternative returner is used then the return will not come back tot he command line @@ -12263,6 +17803,10 @@ but will be sent to the specified return system. .INDENT 0.0 .TP .B \-Q, \-\-query +The \-Q option is being deprecated and will be removed in version 0.9.9, +Use the salt jobs interface instead, for documentation on the salt jobs +interface execute the command "salt\-run \-d jobs" +.sp Execute a salt command query, this can be used to find the results of a previous function call: \-Q test.echo\(aq) .UNINDENT @@ -12296,6 +17840,11 @@ Print the output from the salt command in yaml. .B \-\-json\-out Print the output from the salt command in json. .UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-color +Disable all colored output +.UNINDENT .SS See also .sp \fIsalt(7)\fP @@ -12333,6 +17882,11 @@ Specify user to run minion .UNINDENT .INDENT 0.0 .TP +.B \-\-pid\-file PIDFILE +Specify the location of the pidfile. +.UNINDENT +.INDENT 0.0 +.TP .B \-l LOG_LEVEL, \-\-log\-level=LOG_LEVEL Console log level. One of \fBinfo\fP, \fBnone\fP, \fBgarbage\fP, \fBtrace\fP, \fBwarning\fP, \fBerror\fP, \fBdebug\fP. For the logfile @@ -12371,6 +17925,11 @@ Specify user to run minion .UNINDENT .INDENT 0.0 .TP +.B \-\-pid\-file PIDFILE +Specify the location of the pidfile +.UNINDENT +.INDENT 0.0 +.TP .B \-l LOG_LEVEL, \-\-log\-level=LOG_LEVEL Console log level. One of \fBinfo\fP, \fBnone\fP, \fBgarbage\fP, \fBtrace\fP, \fBwarning\fP, \fBerror\fP, \fBdebug\fP. For the logfile @@ -12423,6 +17982,16 @@ Rejects all pending public keys. .UNINDENT .INDENT 0.0 .TP +.B \-d DELETE, \-\-delete=DELETE +Delete the named minion key for command execution. +.UNINDENT +.INDENT 0.0 +.TP +.B \-D DELETE_ALL, \-\-delete\-all=DELETE_ALL +Deleta all keys +.UNINDENT +.INDENT 0.0 +.TP .B \-c CONFIG, \-\-config=CONFIG The master configuration file needs to be read to determine where the salt keys are stored via the pki_dir configuration value; @@ -12473,14 +18042,34 @@ example: server1.foo.bar,server2.foo.bar,example7.quo.qux .TP .B \-G, \-\-grain The target expression matches values returned by the salt grains system on +the minions. The target expression is in the format of \(aq<grain value>:<glob +expression>\(aq; example: \(aqos:Arch*\(aq +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-grain\-pcre +The target expression matches values returned by the salt grains system on the minions. The target expression is in the format of \(aq<grain value>:<pcre regular expression>\(aq; example: \(aqos:Arch.*\(aq .UNINDENT .INDENT 0.0 .TP -.B \-Q, \-\-query -Execute a salt command query, this can be used to find the results of a -previous function call: \-Q test.echo\(aq) +.B \-R, \-\-range +Instead of using shell globs to evaluate the targe use a range expression +to identify targets. Range expressions look like %cluster. +.sp +Using the Range option requires that a range server is set up and the +location of the range server is referenced in the master configuration +file. +.UNINDENT +.INDENT 0.0 +.TP +.B \-C, \-\-compound +Utilize many target definitions to make the call very granular. This option +takes a group of targets separated by and or or. The default matcher is a +glob as usual, if something other than a glob is used preface it with the +letter denoting the type, example: \(aqwebserv* and \fI\%G@os\fP:Debian or \fI\%E@db*\fP\(aq +make sure that the compound target is encapsulated in quotes. .UNINDENT .INDENT 0.0 .TP @@ -12520,6 +18109,41 @@ directories can be delimited by commas Return the documentation for the specified module of for all modules if none are specified .UNINDENT +.INDENT 0.0 +.TP +.B \-l LOG_LEVEL, \-\-log\-level=LOG_LEVEL +Console log level. One of \fBinfo\fP, \fBnone\fP, \fBgarbage\fP, +\fBtrace\fP, \fBwarning\fP, \fBerror\fP, \fBdebug\fP. For the logfile +settings see the config file. Default: \fBinfo\fP. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-raw\-out +Print the output from the salt command in raw python +form, this is suitable for re\-reading the output into +an executing python script with eval. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-text\-out +Print the output from the salt command in the same +form the shell would. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-yaml\-out +Print the output from the salt command in yaml. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-json\-out +Print the output from the salt command in json. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-color +Disable all colored output +.UNINDENT .SH SALT-RUN .sp Execute a Salt runner @@ -12572,6 +18196,11 @@ Run the salt syndic as a daemon .UNINDENT .INDENT 0.0 .TP +.B \-\-pid\-file PIDFILE +Specify the location of the pidfile +.UNINDENT +.INDENT 0.0 +.TP .B \-\-master\-config=MASTER_CONFIG The master configuration file to use, the default is /etc/salt/master .UNINDENT @@ -12580,6 +18209,93 @@ The master configuration file to use, the default is /etc/salt/master .B \-\-minion\-config=MINION_CONFIG The minion configuration file to use, the default is /etc/salt/minion .UNINDENT +.SH ABSTRACT ON SALT AUTHENTICATION AND ENCRYPTION +.sp +The Salt authentication and encryption system uses Public Key authentication +and AES encryption to facilitate both authentication and high speed encryption. +.sp +The core components of this system can be separated into a few sections, +Message Formatting, PubKey Handshake, AES key management, and encryption. +.SS Message Formatting +.sp +All messages passed within Salt are formatted with a clear header and a "load". +The header is always clear, and specifies the encryption used by the load. The +load can be encrypted with the private key of the sending system, or with the +shared, rotating, AES key. +.sp +The message itself is abstracted as a python dict in this fashion: +.sp +.nf +.ft C +{\(aqenc\(aq: \(aqaes\(aq, + \(aqload\(aq: <encrypted msgpack binary data>} +.ft P +.fi +.sp +When this message is received the load can be decrypted using the shared AES +key. The \(aqenc\(aq dict key can also be "pub" for pubkey encryption, or "clear" +for passing messages in the clear. +.SS PubKey Handshake +.sp +RSA Public keys are generated on the Salt master and on the Salt minion. When +A salt minion establishes an initial connection to the salt master the minion +sends its public key in the clear to the salt master, along with the id of +the minion, and the command to execute on the master, in this case "_auth": +.sp +.nf +.ft C +{\(aqenc\(aq: \(aqclear\(aq, + \(aqload\(aq: + {\(aqcmd\(aq: \(aq_auth\(aq, + \(aqid\(aq: <minion id>, + \(aqpub\(aq: <minion public key>}} +.ft P +.fi +.sp +If this is the first time this minion has authenticated, then the salt master +returns a clear message informing the minion that it is pending authentication. +The minion then queries the master every ten seconds awaiting authentication. +When the public key of the minion has been approved by the master, then the +master\(aqs public key is returned, with the AES key used to encrypt messages and +information on how to connect to the salt master publish interface. +.sp +The returned AES key is encrypted with the minion\(aqs public key, and can +therefore only be decrypted by the minion that sent out the public key. +.sp +Once the minion has authenticated and is in possession of the revolving master +AES key (The AES key is regenerated when the master restarts) then it attaches +the minion subscriber to the master publisher. +.sp +All messages sent from the publisher are encrypted using the revolving AES key, +in the event that the master restarts the minions will all have an invalid +AES key because it has been regenerated on the master. The master will then +send out a publication that the minions cannot decrypt. If the minion receives +a publication that cannot be decrypted then the minion will re\-authenticate, +obtain the correct AES key, and decrypt the message. This means that the +AES key on the salt master can safely revolve without interrupting the minion +connection. +.SS Regular Communication +.sp +Once the minion has authenticated, then all messages sent between the minion +and the master are encrypted using the revolving AES key and the {\(aqenc\(aq: \(aqaes\(aq} +header. +.SS Source Files Implimenting Components +.sp +The pubkey authentication is managed via the salt.master module: +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/master.py\fP +The regular minion authentication is managed via the salt.crypt module: +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/crypt.py\fP +The salt.crypt module contains a class "SAuth" that can be used for +standalone authentication with the Salt master, this is most likely the best +place to start when looking into how the authentication mechanism works +The encrypted "load" portion of the messages are encrypted and decrypted using +the Crypticle class in the crypt module. +.SS Conclusion +.sp +In the end Salt uses formatted messages with clear header data to specify how +the message data is encrypted. Asymetric encryption via RSA keys is only used +for authentication and to securely retrieve the master AES key. All further +communications are are encrypted with 256 bit AES. .SH SALT STACK ROADMAP .sp Salt is the core of a more complete goal, the Salt Stack. The Salt Stack is a @@ -12597,7 +18313,7 @@ appear in the web interface. The features listed here are only listed based on major version releases, the plan is to have a clear long term goal but not to overly dictate the flow of development. The project needs to be flexible enough to be able to receive -now features at a moment\(aqs notice. This model should spur ideas and make +new features at a moment\(aqs notice. This model should spur ideas and make allowing new community developers to join without issue. So just because something is slated for a later reason does not mean that a developer is going to have their code rejected or stalled. @@ -12614,7 +18330,7 @@ Python 3 compatible code. Many instances of using functions when we should be using module cross calls exist in the code. Mostly from modules which were written before cross calls were around. The big thing to look for are subprocess calls, since they should -all be running with the cms module. +all be running with the cmd module. .SS State Return Data Cleanup .sp The return structures in the state modules need to be uniform in how they are @@ -12957,7 +18673,7 @@ use the file extension “.pyx” and the minion module will be compiled when the minion is started. An example cython module is included in the main distribution called cytest.pyx: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/modules/cytest.pyx\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/modules/cytest.pyx\fP .SS Dynamic Returners \- .sp By default salt returns command data back to the salt master, but now salt can @@ -12971,7 +18687,7 @@ data so anything from MySQL, redis, mongodb and more! .sp There are 2 simple stock returners in the returners directory: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/returners\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/returners\fP .sp The documentation on writing returners will be added to the wiki shortly, and returners can be written in pure python, or in cython. @@ -12987,7 +18703,7 @@ Information on how to use this simple addition has been added to the wiki: The test module has an example of using the __opts__ dict, and how to set default options: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/modules/test.py\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/modules/test.py\fP .SS Advanced Minion Threading: .sp In 0.7.0 the minion would block after receiving a command from the master, now @@ -12999,7 +18715,7 @@ exploit the negative aspects of the Python GIL to run faster and more reliably, but simple calls will still be faster with python threading. The configuration option can be found in the minion configuration file: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/conf/minion\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/conf/minion\fP .sp Lowered Supported Python to 2.6 \- .sp @@ -13077,7 +18793,7 @@ The system for loading salt modules has been pulled out of the minion class to be a standalone module, this has enabled more dynamic loading of Salt modules and enables many of the updates in 0.8.7 – .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/loader.py\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/loader.py\fP .sp Salt Job ids are now microsecond precise, this was needed to repair a race condition unveiled by the speed improvements in the new ZeroMQ topology. @@ -13353,7 +19069,7 @@ The minion and master classes have been redesigned to allow for specialized minion and master servers to be easily created. An example on how this is done for the master can be found in the \fBmaster.py\fP salt module: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/master.py\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/master.py\fP .sp The \fBMaster\fP class extends the \fBSMaster\fP class and set up the main master server. @@ -13361,7 +19077,7 @@ server. The minion functions can now also be easily added to another application via the \fBSMinion\fP class, this class can be found in the \fBminion.py\fP module: .sp -\fI\%https://github.com/saltstack/salt/blob/v0.9.7/salt/minion.py\fP +\fI\%https://github.com/saltstack/salt/blob/v0.9.8/salt/minion.py\fP .SS Cleaner Key Management .sp This release changes some of the key naming to allow for multiple master keys @@ -14307,7 +20023,7 @@ under the hood, as well as some features that make working with Salt much better. .sp A few highlights include the new Job system, refinements to the requisite -system in states, the mod_init interface for states, external node +system in states, the \fBmod_init\fP interface for states, external node classification, search path to managed files in the file state, and refinements and additions to dynamic module loading. .sp @@ -14318,35 +20034,29 @@ and the initial unit tests. .sp The new jobs interface makes the management of running executions much cleaner and more transparent. Building on the existing execution framework the jobs -system allows clear introspection into the active running state of the, +system allows clear introspection into the active running state of the running Salt interface. .sp The Jobs interface is centered in the new minion side proc system. The -minions now store a msgpack serialized file under \fBcachedir\fP/proc. These -files keep track of the active state of processes on the minion. +minions now store msgpack serialized files under \fB/var/cache/salt/proc\fP. +These files keep track of the active state of processes on the minion. .SS Functions in the saltutil Module .sp A number of functions have been added to the saltutil module to manage and view the jobs: -.INDENT 0.0 -.IP 1. 3 -running -Returns the data of all running jobs that are found in the proc directory. -.IP 2. 3 -find_job -Returns specific data about a certain job based on job id. -.IP 3. 3 -signal_job -Allows for a given jid to be sent a signal. -.IP 4. 3 -term_job -Sends a termination signal (SIGTERM, 15) to the process controlling the -specified job. -.IP 5. 3 -kill_job -Sends a kill signal (SIGKILL, 9) to the process controlling the +.sp +\fBrunning\fP \- Returns the data of all running jobs that are found in the proc +directory. +.sp +\fBfind_job\fP \- Returns specific data about a certain job based on job id. +.sp +\fBsignal_job\fP \- Allows for a given jid to be sent a signal. +.sp +\fBterm_job\fP \- Sends a termination signal (\fBSIGTERM, 15\fP) to the process +controlling the specified job. +.sp +\fBkill_job\fP Sends a kill signal (\fBSIGKILL, 9\fP) to the process controlling the specified job. -.UNINDENT .SS The jobs Runner .sp A convenience runner front end and reporting system has been added as well. @@ -14355,7 +20065,7 @@ The jobs runner contains functions to make viewing data easier and cleaner. The jobs runner contains a number of functions... .SS active .sp -The active function runs saltutil.running on all minions and formats the +The active function runs \fBsaltutil.running\fP on all minions and formats the return data about all running jobs in a much more usable and compact format. The active function will also compare jobs that have returned and jobs that are still running, making it easier to see what systems have completed a job @@ -14365,13 +20075,14 @@ and what systems are still being waited on. When jobs are executed the return data is sent back to the master and cached. By default is is cached for 24 hours, but this can be configured via the \fBkeep_jobs\fP option in the master configuration. -Using the lookup_jid runner will display the same return data that the initial -job invocation with the salt command would display. +.sp +Using the \fBlookup_jid\fP runner will display the same return data that the +initial job invocation with the salt command would display. .SS list_jobs .sp -Before finding a historic job, it may be required to find the job id. list_jobs -will parse the cached execution data and display all of the job data for jobs -that have already, or partially returned. +Before finding a historic job, it may be required to find the job id. +\fBlist_jobs\fP will parse the cached execution data and display all of the job +data for jobs that have already, or partially returned. .SS External Node Classification .sp Salt can now use external node classifiers like Cobbler\(aqs @@ -14392,14 +20103,14 @@ only use external nodes, do not deploy any top files. An issue arose with the pkg state. Every time a package was run Salt would need to refresh the package database. This made systems with slower package metadata refresh speeds much slower to work with. To alleviate this issue the -mod_init interface has been added to salt states. +\fBmod_init\fP interface has been added to salt states. .sp -The mod_init interface is a function that can be added to a state file. +The \fBmod_init\fP interface is a function that can be added to a state file. This function is called with the first state called. In the case of the pkg -state, the mod_init function sets up a tag which makes the package database +state, the \fBmod_init\fP function sets up a tag which makes the package database only refresh on the first attempt to install a package. .sp -In a nutshell, the mod_init interface allows a state to run any command that +In a nutshell, the \fBmod_init\fP interface allows a state to run any command that only needs to be run once, or can be used to set up an environment for working with the state. .SS Source File Search Path @@ -14434,7 +20145,7 @@ This issue has been alleviated, as well as making the requisite system run more quickly. .SS Initial Unit Testing Framework .sp -Because of the module system, and the need to test real scenarios the, +Because of the module system, and the need to test real scenarios, the development of a viable unit testing system has been difficult, but unit testing has finally arrived. Only a small amount of unit testing coverage has been developed, much more coverage will be in place soon. @@ -14452,6 +20163,312 @@ Previously the nodegroups defined in the master configuration file could not be used to match nodes for states. The nodegroups support has been expanded and the nodegroups defined in the master configuration can now be used to match minions in the top file. +.SS Salt 0.9.8 Release Notes +.sp +Salt 0.9.8 is a big step forward, with many additions and enhancements, as +well as a number of precursors to advanced future developments. +.sp +This version of Salt adds much more power to the command line, making the +old hard timeout issues a thing of the past and adds keyword argument +support. These additions are also available in the salt client api, making +the available api tools much more powerful. +.sp +The new pillar system allows for data to be stored on the master and +assigned to minions in a granular way similar to the state system. It also +allows flexibility for users who want to keep data out of their state tree +similar to \(aqexternal lookup\(aq functionality in other tools. +.sp +Additions to requisites making them much more powerful have been added and +improved error checking for sls files in the state system. A new provider +system has been added to allow for redirecting what modules run in the +background for individual states. +.sp +Support for OpenSUSE has been added and support for Solaris has begun +serious development. Windows support has been significantly enhanced as well. +.sp +The matcher and target systems have received a great deal of attention. The +default behavior of grain matching has changed slightly to reflect the rest +of salt and the compound matcher system has been refined. +.sp +A number of impressive features with keyword arguments have been added to both +the cli and to the state system. This makes states much more powerful and +flexible while maintaining the simple configuration everyone loves. +.sp +The new batch size capability allows for executions to be rolled through a +group of targeted minions a percentage or specific number at a time. This +was added to prevent the "thundering herd" problem when targeting large +numbers of minions for things like service restarts or file downloads. +.SS Upgrade Considerations +.SS Upgrade Issues +.sp +There was a previously missed oversight which could cause a newer minion to +crash an older master. That oversight has been resolved so the version +incompatibility issue will no longer occur. When upgrading to 0.9.8 make +sure to upgrade the master first, followed by the minions. +.SS Debian/Ubuntu Packages +.sp +The original Debian/Ubuntu packages were called salt and included all salt +applications. New packages in the ppa are split by function. If an old salt +package is installed then it should be manually removed and the new split +packages need to be freshly installed. +.sp +On the master: +.sp +.nf +.ft C +# apt\-get purge salt +# apt\-get install salt\-{master,minion} +.ft P +.fi +.sp +On the minions: +.sp +.nf +.ft C +# apt\-get purge salt +# apt\-get install salt\-minion +.ft P +.fi +.sp +And on any Syndics: +.sp +.nf +.ft C +# apt\-get install salt\-syndic +.ft P +.fi +.sp +The official salt stack ppa for Ubuntu is located at: +\fI\%https://launchpad.net/~saltstack/+archive/salt\fP +.SS Major Features +.SS Pillar +.sp +\fBPillar\fP offers an interface to declare variable data on the master that is then +assigned to the minions. The pillar data is made available to all modules, +states, sls files etc. It is compiled on the master and is declared using the +existing renderer system. This means that learning pillar should be fairly +trivial to those already familiar with salt states. +.SS CLI Additions +.sp +The \fBsalt\fP command has received a serious overhaul and is more powerful +than ever. Data is returned to the terminal as it is received, and the salt +command will now wait for all running minions to return data before stopping. +This makes adding very large \fI\-\-timeout\fP arguments completely unnecessary and +gets rid of long running operations returning empty \fB{}\fP when the timeout is +exceeded. +.sp +When calling salt via sudo, the user originally running salt is saved to the +log for auditing purposes. This makes it easy to see who ran what by just +looking through the minion logs. +.sp +The \fIsalt\-key\fP command gained the \fI\-D\fP and \fI\-\-delete\-all\fP arguments for +removing all keys. Be careful with this one! +.SS Keyword Arguments and States +.sp +State modules now accept the \fB**kwargs\fP argument. This results in all data +in a sls file assigned to a state will be made available to the state function. +.sp +This passes data in a transparent way back to the modules executing the logic. +In particular, this allows adding arguments to the \fBpkg.install\fP module that +enable more advanced and granular controls with respect to what the state is +capable of. +.sp +An example of this along with the new debconf module for installing ldap +client packages on Debian: +.sp +.nf +.ft C +ldap\-client\-packages: + pkg: + \- debconf: salt://debconf/ldap\-client.ans + \- installed + \- names: + \- nslcd + \- libpam\-ldapd + \- libnss\-ldapd +.ft P +.fi +.SS Keyword Arguments and the CLI +.sp +In the past it was required that all arguments be passed in the proper order to +the \fIsalt\fP and \fIsalt\-call\fP commands. As of 0.9.8, keyword arguments can be +passed in the form of \fBkwarg=argument\fP. +.sp +.nf +.ft C +# salt \-G \(aqtype:dev\(aq git.clone \e + repository=https://github.com/saltstack/salt.git cwd=/tmp/salt user=jeff +.ft P +.fi +.SS Matcher Refinements and Changes +.sp +A number of fixes and changes have been applied to the Matcher system. The +most noteworthy is the change in the grain matcher. The grain matcher used +a regular expression to match the passed data to a grain, but now defaults +to a shell glob like the majority of match interfaces in Salt. A new option +is available that still uses the old style regex matching to grain data called +grain\-pcre. To use regex matching in compound matches use the letter \fIP\fP. +.sp +For example, this would match any ArchLinux or Fedora minions: +.sp +.nf +.ft C +# salt \-\-grain\-pcre \(aqos:(Arch:Fed).*\(aq test.ping +.ft P +.fi +.sp +And the associated compound matcher suitable for \fBtop.sls\fP is \fIP\fP: +.sp +.nf +.ft C +P@os:(Arch|Fed).* +.ft P +.fi +.sp +\fBNOTE\fP: Changing the grains matcher from pcre to glob is backwards +incompatible. +.sp +Support has been added for matching minions with Yahoo\(aqs range library. This +is handled by passing range syntax with \fI\-R\fP or \fI\-\-range\fP arguments to salt. +.sp +More information at: +\fI\%https://github.com/grierj/range/wiki/Introduction-to-Range-with-YAML-files\fP +.SS Providers +.sp +Salt predetermines what modules should be mapped to what uses based on the +properties of a system. These determinations are generally made for modules +that provide things like package and service management. The apt module +maps to pkg on Debian and the yum module maps to pkg on Fedora for instance. +.sp +Sometimes in states, it may be necessary for a non\-default module to be used +for the desired functionality. For instance, an Arch Linux system may have +been set up with systemd support. Instead of using the default service module +detected for Arch Linux, the systemd module can be used: +.sp +.nf +.ft C +http: + service: + \- running + \- enable: True + \- provider: systemd +.ft P +.fi +.SS Requisite Glob Matching +.sp +Requisites can now be defined with glob expansion. This means that if there are +many requisites, they can be defined on a single line. +.sp +To watch all files in a directory: +.sp +.nf +.ft C +http: + service: + \- running + \- enable: True + \- watch: + \- file: /etc/http/conf.d/* +.ft P +.fi +.sp +This example will watch all defined files that match the glob +\fB/etc/http/conf.d/*\fP +.SS Batch Size +.sp +The new batch size option allows commands to be executed while maintaining that +only so many hosts are executing the command at one time. This option can +take a percentage or a finite number: +.sp +.nf +.ft C +salt \e* \-b 10 test.ping + +salt \-G \(aqos:RedHat\(aq \-\-batch\-size 25% apache.signal restart +.ft P +.fi +.sp +This will only run test.ping on 10 of the targeted minions at a time and then +restart apache on 25% of the minions matching \fBos:RedHat\fP at a time and work +through them all until the task is complete. This makes jobs like rolling web +server restarts behind a load balancer or doing maintenance on BSD firewalls +using carp much easier with salt. +.SS Module Updates +.sp +This is a list of notable, but non\-exhaustive updates with new and existing +modules. +.sp +Windows support has seen a flurry of support this release cycle. We\(aqve gained +all new \fBfile\fP, +\fBnetwork\fP, and +\fBshadow\fP modules. Please note +that these are still a work in progress. +.sp +For our ruby users, new \fBrvm\fP and +\fBgem\fP modules have been added along +with the \fBassociated\fP +\fBstates\fP +.sp +The \fBvirt\fP module gained basic Xen support. +.sp +The \fByum\fP +\fBpkg\fP modules gained Scientific +Linux support. +.sp +The \fBpkg\fP module on Debian, Ubuntu, +and derivatives force apt to run in a non\-interactive mode. This prevents +issues when package installation waits for confirmation. +.sp +A \fBpkg\fP module for OpenSUSE\(aqs +zypper was added. +.sp +The \fBservice\fP module on ubuntu +natively supports upstart. +.sp +A new \fBdebconf\fP module was +contributed by our community for more advanced control over deb package +deployments on Debian based distributions. +.sp +The \fBmysql.user\fP state and +\fBmysql\fP module gained a +\fIpassword_hash\fP argument. +.sp +The \fBcmd\fP module and state gained +a \fIshell\fP keyword argument for specifying a shell other than \fB/bin/sh\fP on +Linux / Unix systems. +.sp +New \fBgit\fP and +\fBmercurial\fP modules have been added +for fans of distributed version control. +.SS In Progress Development +.SS Master Side State Compiling +.sp +While we feel strongly that the advantages gained with minion side state +compiling are very critical, it does prevent certain features what may be +desired. 0.9.8 has support for initial master side state compiling, but many +more components still need to be developed, it is hoped that these can be +finished for 0.9.9. +.sp +The goal is that states can be compiled on both the master and the minion +allowing for compilation to be split between master and minion. Why will +this be great? It will allow storing sensitive data on the master and sending +it to some minions without all minions having access to it. This will be +good for handling ssl certificates on front\-end web servers. +.SS New File Client +.sp +The file client code has been re\-factored to allow local \fBsalt://\fP uris. +This will eventually allow for running salt\(aqs configuration management via +\fBsalt\-call state.highstate\fP to run without a salt\-minion daemon running. +.SS Solaris Support +.sp +Salt 0.9.8 sees the introduction of basic Solaris support. The daemon runs +well, but grains and more of the modules need updating and testing. +.SS Windows Support +.sp +Salt states on windows are now much more viable thanks to contributions from +our community! States for file, user, and group management are more fully +fleshed out along with a network module. Windows users can also now manage +registry entries using the new "reg" module. .SH AUTHOR Thomas S. Hatch <thatch@gmail.com> and many others, please see the Authors file .SH COPYRIGHT From 24c97295f6fbd6289f25a9f28922d35f07b28b10 Mon Sep 17 00:00:00 2001 From: blast_hardcheese <blast@hardchee.se> Date: Mon, 19 Mar 2012 13:42:46 -0700 Subject: [PATCH 541/598] The apt module now has a non-standard external dep Before this note gets lost in time, let's document it. --- salt/modules/apt.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/modules/apt.py b/salt/modules/apt.py index bdfefe1db45b..a69e5fd9a76f 100644 --- a/salt/modules/apt.py +++ b/salt/modules/apt.py @@ -251,9 +251,15 @@ def list_pkgs(regex_string=""): {'<package_name>': '<version>'} + External dependencies:: + + Virtual package resolution requires aptitude. + Without aptitude virtual packages will be reported as not installed. + CLI Example:: salt '*' pkg.list_pkgs + salt '*' pkg.list_pkgs httpd ''' ret = {} cmd = 'dpkg --list {0}'.format(regex_string) From 1655ab59e0417800e9802a9fd561ed2f21b1ad07 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 15:30:42 -0600 Subject: [PATCH 542/598] fixed this issue where using a dot is now much cleaner --- doc/topics/tutorials/states_pt1.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/topics/tutorials/states_pt1.rst b/doc/topics/tutorials/states_pt1.rst index 04a78699a4d6..58b5544c7d7b 100644 --- a/doc/topics/tutorials/states_pt1.rst +++ b/doc/topics/tutorials/states_pt1.rst @@ -86,10 +86,6 @@ In this case it defines the name of the package to be installed. **NOTE:** the package name for the Apache httpd web server may differ on your OS or distro — for example, on Fedora it is ``httpd`` but on Debian/Ubuntu it is ``apache2``. -Additionally, an ID declaration should not contain a dot, as this will produce -unpredictable output in the summary returned from a call to -:func:`state.highstate <salt.modules.state.highstate>`. - The second line, called the :term:`state declaration`, defines which of the Salt States we are using. In this example, we are using the :mod:`pkg state <salt.states.pkg>` to ensure that a given package is installed. From a19661710014f66454175d4e75965cea4d9da8c8 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 15:45:44 -0600 Subject: [PATCH 543/598] Add -n to freebsd pw usermod --- salt/modules/pw_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/pw_user.py b/salt/modules/pw_user.py index 0a4fe7c1b98f..9c47130bbc38 100644 --- a/salt/modules/pw_user.py +++ b/salt/modules/pw_user.py @@ -175,7 +175,7 @@ def chgroups(name, groups, append=False): ugrps = set(list_groups(name)) if ugrps == set(groups): return True - cmd = 'pw usermod -G {0} {1} '.format(','.join(groups), name) + cmd = 'pw usermod -G {0} -n {1} '.format(','.join(groups), name) if append: cmd += '-a' __salt__['cmd.run'](cmd) From 47915e7309fdc4cb76447bd35daaeb0055a88ee1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 16:12:23 -0600 Subject: [PATCH 544/598] Add **kwargs to pw_user to absorb extra data sent from state --- salt/modules/pw_user.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/pw_user.py b/salt/modules/pw_user.py index 9c47130bbc38..378919273c5e 100644 --- a/salt/modules/pw_user.py +++ b/salt/modules/pw_user.py @@ -19,7 +19,8 @@ def add(name, gid=None, groups=None, home=True, - shell=None): + shell=None, + **kwargs): ''' Add a user to the minion From 9600435b921d5722c764ae307925f1dd8506d886 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 16:28:57 -0600 Subject: [PATCH 545/598] Repair break in salt-call -g --- salt/cli/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 9faead748e66..13fedc6e90fc 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -711,6 +711,8 @@ def __parse(self): if len(args) >= 1: opts['fun'] = args[0] opts['arg'] = args[1:] + elif opts['grains']: + pass else: # salt-call should not ever be called without arguments parser.print_help() From 82426ba9a4acb605e1df71e47da01a729c7bee60 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 16:30:35 -0600 Subject: [PATCH 546/598] Correct repair of salt-call -g --- salt/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 13fedc6e90fc..3dde5c4ae153 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -711,7 +711,7 @@ def __parse(self): if len(args) >= 1: opts['fun'] = args[0] opts['arg'] = args[1:] - elif opts['grains']: + elif opts['grains_run']: pass else: # salt-call should not ever be called without arguments From 038d040588d319ae5bc9757aaec3186145d9629a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 16:43:19 -0600 Subject: [PATCH 547/598] make the user state pass kwargs --- salt/states/user.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/states/user.py b/salt/states/user.py index 46c13aab40d6..1e3954f4f70e 100644 --- a/salt/states/user.py +++ b/salt/states/user.py @@ -167,8 +167,17 @@ def present( return ret # The user is not present, make it! - if __salt__['user.add'](name, uid, gid, groups, home, shell, fullname, - roomnumber, workphone, homephone, other): + if __salt__['user.add'](name, + uid=uid, + gid=gid, + groups=groups, + home=home, + shell=shell, + fullname=fullname, + roomnumber=roomnumber, + workphone=workphone, + homephone=homephone, + other=other): ret['comment'] = 'New user {0} created'.format(name) ret['changes'] = __salt__['user.info'](name) if password: From ba773570280e64780c0497f12381437ae996caf7 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 17:30:43 -0600 Subject: [PATCH 548/598] change user to create homedir by default --- salt/states/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/user.py b/salt/states/user.py index 1e3954f4f70e..7f359552687a 100644 --- a/salt/states/user.py +++ b/salt/states/user.py @@ -26,7 +26,7 @@ def present( uid=None, gid=None, groups=None, - home=False, + home=True, password=None, enforce_password=True, shell=None, From ca002556b69cb7e629725e4754a81f0b21e134d0 Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Mon, 19 Mar 2012 21:19:49 -0600 Subject: [PATCH 549/598] Add a little to the Windows release notes. --- doc/topics/releases/0.9.8.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index c4196d204b58..0d36be85f5a9 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -327,6 +327,6 @@ Windows Support -------------------- Salt states on windows are now much more viable thanks to contributions from -our community! States for file, user, and group management are more fully -fleshed out along with a network module. Windows users can also now manage -registry entries using the new "reg" module. +our community! States for file, service, local user, and local group management are more fully +fleshed out along with network and disk modules. Windows users can also now manage +registry entries using the new "reg" module. From 997cf2718535217411657462be4581ceb85fd36a Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Mon, 19 Mar 2012 19:10:41 -0700 Subject: [PATCH 550/598] Very minor refactor of the file module --- salt/modules/file.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 9caa451e4042..6cb4ca04544e 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -20,18 +20,14 @@ def __virtual__(): ''' Only work on posix-like systems ''' - - # Disable on these platforms, specific file modules exist: - disable = [ - 'Windows', - ] - if __grains__['os'] in disable: + # win_file takes care of windows + if __grains__['os'] == 'Windows' return False return 'file' - + __outputter__ = { - 'touch': 'txt', + 'touch': 'txt', 'append': 'txt', } From 1fc4bf12024f48223191d689f15363ea1c5f5ce6 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Mon, 19 Mar 2012 19:21:23 -0700 Subject: [PATCH 551/598] Very minor refactor and cleanup of the 'cmd' module --- salt/modules/cmdmod.py | 43 +++++++++++++++++++++++++----------------- salt/modules/file.py | 2 +- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index fbbcde95dc89..a69a9c54dd81 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -141,7 +141,7 @@ def run(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL, env=()): ''' out = _run(cmd, runas=runas, shell=shell, cwd=cwd, stderr=subprocess.STDOUT, env=env)['stdout'] - log.debug(out) + log.debug('output: {0}'.format(out)) return out @@ -154,7 +154,7 @@ def run_stdout(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL, env=()): salt '*' cmd.run_stdout "ls -l | awk '/foo/{print $2}'" ''' stdout = _run(cmd, runas=runas, cwd=cwd, shell=shell, env=())["stdout"] - log.debug(stdout) + log.debug('stdout: {0}'.format(stdout)) return stdout @@ -167,7 +167,7 @@ def run_stderr(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL, env=()): salt '*' cmd.run_stderr "ls -l | awk '/foo/{print $2}'" ''' stderr = _run(cmd, runas=runas, cwd=cwd, shell=shell, env=env)["stderr"] - log.debug(stderr) + log.debug('stderr: {0}'.format(stderr)) return stderr @@ -181,13 +181,20 @@ def run_all(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL, env=()): ''' ret = _run(cmd, runas=runas, cwd=cwd, shell=shell, env=env) if ret['retcode'] != 0: - log.error('Command {0} failed'.format(cmd)) - log.error('retcode: {0}'.format(ret['retcode'])) - log.error('stdout: {0}'.format(ret['stdout'])) - log.error('stderr: {0}'.format(ret['stderr'])) + rcode = ret['retcode'] + msg = 'Command \'{0}\' failed with return code: {1}' + log.error(msg.format(cmd, rcode)) + # Don't log a blank line if there is no stderr or stdout + if ret['stdout']: + log.error('stdout: {0}'.format(ret['stdout'])) + if ret['stderr']: + log.error('stderr: {0}'.format(ret['stderr'])) else: - log.info('stdout: {0}'.format(ret['stdout'])) - log.info('stderr: {0}'.format(ret['stderr'])) + # No need to always log output on success to the logs + if ret['stdout']: + log.debug('stdout: {0}'.format(ret['stdout'])) + if ret['stderr']: + log.debug('stderr: {0}'.format(ret['stderr'])) return ret @@ -202,25 +209,27 @@ def retcode(cmd, cwd=None, runas=None, shell=DEFAULT_SHELL, env=()): return _run(cmd, runas=runas, cwd=cwd, shell=shell, env=env)['retcode'] -def has_exec(cmd): +def which(cmd): ''' - Returns true if the executable is available on the minion, false otherwise + Returns the path of an executable available on the minion, None otherwise CLI Example:: - salt '*' cmd.has_exec cat + salt '*' cmd.which cat ''' - return bool(salt.utils.which(cmd)) + return salt.utils.which(cmd) -def which(cmd): + +def has_exec(cmd): ''' - Returns the path of an executable available on the minion, None otherwise + Returns true if the executable is available on the minion, false otherwise CLI Example:: - salt '*' cmd.which cat + salt '*' cmd.has_exec cat ''' - return salt.utils.which(cmd) + return bool(which(cmd)) + def exec_code(lang, code, cwd=None): ''' diff --git a/salt/modules/file.py b/salt/modules/file.py index 6cb4ca04544e..7ec73e1822fd 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -21,7 +21,7 @@ def __virtual__(): Only work on posix-like systems ''' # win_file takes care of windows - if __grains__['os'] == 'Windows' + if __grains__['os'] == 'Windows': return False return 'file' From 7d657ef413accbf3c25fb12fd1d063e3b190e8ba Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Mon, 19 Mar 2012 19:32:04 -0700 Subject: [PATCH 552/598] Properly create empty files in file.managed when source=None --- salt/states/file.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 948e9acb35d7..466708833f46 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -761,19 +761,29 @@ def managed(name, else: __clean_tmp(sfn) return _error(ret, 'Parent directory not present') - # Create the file, user-rw-only if mode will be set + + # Create the file, user rw-only if mode will be set to prevent + # a small security race problem before the permissions are set if mode: - cumask = os.umask(384) + current_umask = os.umask(077) + + # Create a new file when test is False and source is None + if not __opts__['test']: + if __salt__['file.touch'](name): + ret['changes']['new'] = 'file {0} created'.format(name) + ret['comment'] = 'Empty file' + else: + return _error(ret, 'Empty file {0} not created'.format(name)) + if mode: - os.umask(cumask) - ret['changes']['new'] = 'file {0} created'.format(name) - ret['comment'] = 'Empty file' + os.umask(current_umask) # Now copy the file contents if there is a source file if sfn: shutil.copyfile(sfn, name) __clean_tmp(sfn) + # Check and set the permissions if necessary ret, perms = _check_perms(name, ret, user, group, mode) if not ret['comment']: From 545dca075186c6c7d0d11cada5327de2c41a4f09 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder <jeffschroeder@computer.org> Date: Mon, 19 Mar 2012 21:05:26 -0700 Subject: [PATCH 553/598] Fix a issue with multi-value sysctls Following state file to reproduce: net.ipv4.tcp_rmem: sysctl: - present - value: 4096 87380 16777216 net.ipv4.tcp_wmem: sysctl: - present - value: 4096 87380 16777216 Without this patch, these two sysctls will be constantly set over and over again each time state.highstate is called --- salt/modules/linux_sysctl.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/salt/modules/linux_sysctl.py b/salt/modules/linux_sysctl.py index 679967aa9b31..b4d87492686c 100644 --- a/salt/modules/linux_sysctl.py +++ b/salt/modules/linux_sysctl.py @@ -8,6 +8,7 @@ __outputter__ = { 'assign': 'txt', + 'get': 'txt', } @@ -106,9 +107,21 @@ def persist(name, value, config='/etc/sysctl.conf'): if '=' not in line: nlines.append(line) continue - comps = line.split('=') - comps[0] = comps[0].strip() - comps[1] = comps[1].strip() + + # Strip trailing whitespace and split the k,v + comps = [i.strip() for i in line.split('=', 1)] + + # On Linux procfs, files such as /proc/sys/net/ipv4/tcp_rmem or any + # other sysctl with whitespace in it consistently uses 1 tab. Lets + # allow our users to put a space or tab between multi-value sysctls + # and have salt not try to set it every single time. + if isinstance(comps[1], basestring) and ' ' in comps[1]: + comps[1] = re.sub('\s+', '\t', comps[1]) + + # Do the same thing for the value 'just in case' + if isinstance(value, basestring) and ' ' in value: + value = re.sub('\s+', '\t', value) + if len(comps) < 2: nlines.append(line) continue From 345a6383e14655938714b61ef62e4e2ae4231f1c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Mon, 19 Mar 2012 22:16:15 -0600 Subject: [PATCH 554/598] fix typo in man pages on Range target --- doc/ref/cli/salt-cp.rst | 2 +- doc/ref/cli/salt.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ref/cli/salt-cp.rst b/doc/ref/cli/salt-cp.rst index 6f47a2316bca..a72bb1ee2d17 100644 --- a/doc/ref/cli/salt-cp.rst +++ b/doc/ref/cli/salt-cp.rst @@ -58,7 +58,7 @@ Options .. option:: -R, --range - Instead of using shell globs to evaluate the targe use a range expression + Instead of using shell globs to evaluate the target use a range expression to identify targets. Range expressions look like %cluster. Using the Range option requires that a range server is set up and the diff --git a/doc/ref/cli/salt.rst b/doc/ref/cli/salt.rst index 5ae1bfab7fd8..bd7dcce8524f 100644 --- a/doc/ref/cli/salt.rst +++ b/doc/ref/cli/salt.rst @@ -98,7 +98,7 @@ Options .. option:: -R, --range - Instead of using shell globs to evaluate the targe use a range expression + Instead of using shell globs to evaluate the target use a range expression to identify targets. Range expressions look like %cluster. Using the Range option requires that a range server is set up and the From eab62c2cf8207e10c2ad1139159d57d64b000351 Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Mon, 19 Mar 2012 21:09:35 -0600 Subject: [PATCH 555/598] First Windows Installation page --- doc/topics/installation/index.rst | 1 + doc/topics/installation/windows.rst | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 doc/topics/installation/windows.rst diff --git a/doc/topics/installation/index.rst b/doc/topics/installation/index.rst index 7324e27635eb..e46f9902e0ef 100644 --- a/doc/topics/installation/index.rst +++ b/doc/topics/installation/index.rst @@ -45,3 +45,4 @@ Platform-specific installation instructions fedora freebsd gentoo + windows diff --git a/doc/topics/installation/windows.rst b/doc/topics/installation/windows.rst new file mode 100644 index 000000000000..d13ad201533c --- /dev/null +++ b/doc/topics/installation/windows.rst @@ -0,0 +1,24 @@ +========== +Windows +========== + +Salt currently has experimental support for Salt Minions on Windows. + +There are no plans for the forseeable future to develop a Salt +Master on Windows. For now you must run your Salt Master on a +supported operating system to control your Salt Minions on Windows. + +Many of the standard Salt modules have been ported to work on Windows +and many of the Salt States currently work on Windows, as well. + +Installation +============ + +Work is under way to create a Windows installer for Salt, but for now +one must install each dependency separately and configure Salt to +run on your Windows host. + +Rather than send you on a wild goose chase across the internet, we've +collected most of the dependency installers in our github repo for you. + + From ba09db25a9383c61736119a8fe772648027ef9cc Mon Sep 17 00:00:00 2001 From: David Boucha <boucha@gmail.com> Date: Tue, 20 Mar 2012 00:59:24 -0600 Subject: [PATCH 556/598] Updated Windows Installation docs This worked well for me. I need some people to attempt to follow these directions. --- doc/topics/installation/windows.rst | 126 +++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 2 deletions(-) diff --git a/doc/topics/installation/windows.rst b/doc/topics/installation/windows.rst index d13ad201533c..8490461678b0 100644 --- a/doc/topics/installation/windows.rst +++ b/doc/topics/installation/windows.rst @@ -11,7 +11,7 @@ supported operating system to control your Salt Minions on Windows. Many of the standard Salt modules have been ported to work on Windows and many of the Salt States currently work on Windows, as well. -Installation +Installation from source ============ Work is under way to create a Windows installer for Salt, but for now @@ -19,6 +19,128 @@ one must install each dependency separately and configure Salt to run on your Windows host. Rather than send you on a wild goose chase across the internet, we've -collected most of the dependency installers in our github repo for you. +collected some of the more difficult to find installers in our github repo for you. +Install on Windows XP 32bit +=========================== +1. Install `msysgit`_ + + 1. Clone the Salt git repository from github + +.. code-block:: bash + + git clone git://github.com/saltstack/salt.git + +2. Install `Microsoft Visual Studio 2008 Express`_ with the web installer. + Or `download a full iso with the installer`_ . + You must use Visual Studio 2008 Express, **not** Visual Studio 2010 Express. + +3. Install `Python 2.7.x`_ + +4. Add c:\\Python27 to your system path + +5. Install the Microsoft Visuall C++ 2008 SP1 Redistributable, `vcredist_x86`_. + +6. Install `Win32OpenSSL-1_0_0e.exe`_ + + #. Choose first option to install in Windows system directory + +7. Install `pyzmq-2.1.11.win32-py2.7.msi`_ + +8. Install `M2Crypto-0.21.1.win32-py2.7.msi`_ + +9. Install `pycrypto-2.3.win32-py2.7.msi`_ + +10. Install `PyYAML-3.10.win32-py2.7.msi`_ + +11. Install `Cython-0.15.1.win32-py2.79.exe`_ + +12. Download and run `distribute_setup.py`_ + +.. code-block:: bash + + python distribute_setup.py + +13. Download and run `pip`_ + +.. code-block:: bash + + python get-pip.py + +14. Add c:\\python27\\scripts to your path + +15. Close terminal window and open a new terminal window (cmd) + +16. Install jinja2 + +.. code-block:: bash + + pip install jinja2 + +17. Install Messagepack + +.. code-block:: bash + + pip install msgpack-python + +18. Install Salt + +.. code-block:: bash + + cd ./salt + python setup.py install + +19. Edit c:\\etc\\salt\\minon + +.. code-block:: bash + + master: ipaddress or hostname of your salt-master + master_port: 4506 + root_dir: c:\ + pki_dir: /etc/salt/pki + cachedir: /var/cache/salt + renderer: yaml_jinja + open_mode: False + multiprocessing: False + +20. Start the salt-minion + +.. code-block:: bash + + cd c:\python27\scripts + python salt-minion + +21. On the salt-master accept the new minion's key + +.. code-block:: bash + + sudo salt-key -A + + (This accepts all unaccepted keys. If you're concerned about security just accept the key for this specific minion) + +22. Test that your minion is responding + + a. On the salt-master run: + +.. code-block:: bash + + sudo salt '*' test.ping + + + You should get the following response: {'your minion hostname': True} + + +.. _msysgit: http://code.google.com/p/msysgit/downloads/list?can=3 +.. _Microsoft Visual Studio 2008 Express: http://www.microsoft.com/visualstudio/en-us/products/2008-editions/express +.. _download a full iso with the installer: http://www.microsoft.com/download/en/details.aspx?id=20682 +.. _Python 2.7.x: http://www.python.org +.. _vcredist_x86: http://www.microsoft.com/download/en/details.aspx?id=5582 +.. _Win32OpenSSL-1_0_0e.exe: http://www.slproweb.com/products/Win32OpenSSL.html +.. _pyzmq-2.1.11.win32-py2.7.msi: https://github.com/zeromq/pyzmq/downloads +.. _M2Crypto-0.21.1.win32-py2.7.msi: http://chandlerproject.org/Projects/MeTooCrypto#Downloads +.. _pycrypto-2.3.win32-py2.7.msi: http://www.voidspace.org.uk/python/modules.shtml#pycrypto +.. _PyYAML-3.10.win32-py2.7.msi: http://pyyaml.org/wiki/PyYAML +.. _Cython-0.15.1.win32-py2.79.exe: http://www.lfd.uci.edu/~gohlke/pythonlibs/#cython +.. _distribute_setup.py: http://python-distribute.org/distribute_setup.py +.. _pip: https://raw.github.com/pypa/pip/master/contrib/get-pip.py From 9d6757f5c3aae6ac3691deee78f4d3e6b7e8b324 Mon Sep 17 00:00:00 2001 From: Nate Smith <nathaniel.smith@thescore.com> Date: Tue, 20 Mar 2012 09:09:50 -0400 Subject: [PATCH 557/598] Work around Python 2.6 bug involving relative paths from root * In Python 2.6, calling `os.path.relpath('/foo/bar', '/')` incorrectly returns a path `../foo/bar`. In Python 2.7 this is fixed, but this commit adds a workaround that works in both 2.6 and 2.7. --- salt/fileclient.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index 9c30b7b4cf8c..653fd0cfcd32 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -249,14 +249,13 @@ def get_url(self, url, dest, makedirs=False, env='base'): else: return '' else: - dest = os.path.join( - self.opts['cachedir'], - 'extrn_files', - env, - os.path.join( + dest = os.path.normpath( + os.sep.join([ + self.opts['cachedir'], + 'extrn_files', + env, url_data.netloc, - os.path.relpath(os.path.relpath(url_data.path, '/'), '..') - )) + url_data.path])) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) From 6f50b6e2edd70e6d983d8d32f49273ef902b81dd Mon Sep 17 00:00:00 2001 From: Andrew Kuhnhausen <trane@xmission.com> Date: Tue, 20 Mar 2012 11:19:50 -0600 Subject: [PATCH 558/598] Add eliptic dsa encoding to ssh module Eliptic dsa encoding has different properties than rsa/dsa. Its encoding string begins with ecdsa-sha2-nistp{256,384,521}. --- salt/modules/ssh.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index b9d529d25612..ce980575c344 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -8,14 +8,24 @@ def _refine_enc(enc): ''' Return the properly formatted ssh value for the authorized encryption key - type. If the type is not found, return ssh-rsa, the ssh default. + type. ecdsa defaults to 256 bits, must give full ecdsa enc schema string if + using higher enc. If the type is not found, return ssh-rsa, the ssh default. ''' - rsa = ['r', 'rsa', 'ssh-rsa'] - dss = ['d', 'dsa', 'dss', 'ssh-dss'] + rsa = ['r', 'rsa', 'ssh-rsa'] + dss = ['d', 'dsa', 'dss', 'ssh-dss'] + ecdsa = ['e', 'ecdsa', 'ecdsa-sha2-nistp521', 'ecdsa-sha2-nistp384', + 'ecdsa-sha2-nistp256'] + if enc in rsa: return 'ssh-rsa' elif enc in dss: return 'ssh-dss' + elif enc in ecdsa: + # ecdsa defaults to ecdsa-sha2-nistp256 + # otherwise enc string is actual encoding string + if enc in ['e', 'ecdsa'] + return 'ecdsa-sha2-nistp256' + return enc else: return 'ssh-rsa' From 00bbcd25b9f8c95833a5766562f5a62ae72a9838 Mon Sep 17 00:00:00 2001 From: Andrew Kuhnhausen <trane@xmission.com> Date: Tue, 20 Mar 2012 11:26:23 -0600 Subject: [PATCH 559/598] Add checks to ecdsa in other methods ecdsa does not begin with ssh- like rsa/dsa. --- salt/modules/ssh.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index ce980575c344..6df5899ea27c 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -74,7 +74,7 @@ def _replace_auth_key( lines.append(line) continue key_ind = 1 - if not comps[0].startswith('ssh-'): + if comps[0][:4:] not in ['ssh-', 'ecds']: key_ind = 2 if comps[key_ind] == key: lines.append(auth_line) @@ -142,7 +142,7 @@ def _validate_keys(key_file): if len(comps) < 2: # Not a valid line continue - if not comps[0].startswith('ssh-'): + if comps[0][:4:] not in ['ssh-', 'ecds']: # It has options, grab them options = comps[0].split(',') else: @@ -190,7 +190,7 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): # Not a valid line lines.append(line) continue - if not comps[0].startswith('ssh-'): + if comps[0][:4:] not in ['ssh-', 'ecds']: # It has options, grab them options = comps[0].split(',') else: From 1a9977693ad24ce65ab4ad8fef7977122d429c0f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 11:31:04 -0600 Subject: [PATCH 560/598] Initial addition of requisite in statements - not pugged in yet! This is the initial and barely tested commit for this feature, more to come, but this allows for a "requisite in" statement. This makes it possible to extend other requisites without making the big extend block. It looks like this: http: service: - running pkg: - installed - watch_in: - service: http make sure to include things that you req_in! --- salt/state.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/salt/state.py b/salt/state.py index fe1099868f34..f236d4269b1a 100644 --- a/salt/state.py +++ b/salt/state.py @@ -591,6 +591,78 @@ def reconcile_extend(self, high): high[name][state].append(arg) return high, errors + def requisite_in(self, high): + ''' + Extend the data reference with requisite_in arguments + ''' + req_in = set('require_in', 'watch_in') + extend = {} + for id_, body in high.items(): + for state, run in body.items(): + if state.startswith('__'): + continue + for arg in run: + if isinstance(arg, dict): + # It is not a function, verify that the arg is a + # requisite in statement + if len(arg) < 1: + # Empty arg dict + # How did we get this far? + continue + # Split out the components + key = arg.keys()[0] + if not key in req_in: + continue + rkey = key[:-3] + items = arg[key] + if isinstance(items, dict): + # Formated as a single req_in + for _state, name in items.items(): + found = False + if not name in extend: + extend[name] = {} + if not _state in extend[name]: + extend[name][_state] = [] + for ind in range(len(extend[name][_state])): + if extend[name][_state][ind].keys()[0] == rkey: + # Extending again + extend[name][_state][ind].append( + {state: id_} + ) + found = True + if found: + continue + # The rkey is not present yet, create it + extend[name][_state].append( + {rkey: [{state: id_}]} + ) + if isinstance(items, list): + # Formed as a list of requisite additions + for ind in items: + if not isinstance(ind, dict): + # Malformed req_in + continue + if len(ind) < 1: + continue + _state = ind.keys()[0] + name = ind[_state] + found = False + for ind in range(len(extend[name][_state])): + if extend[name][_state][ind].keys()[0] == rkey: + # Extending again + extend[name][_state][ind].append( + {state: id_} + ) + found = True + if found: + continue + # The rkey is not present yet, create it + extend[name][_state].append( + {rkey: [{state: id_}]} + ) + high['__extend__'] = extend + return self.reconcile_extend(high) + def call(self, data): ''' Call a state directly with the low data structure, verify data From a6736ba4530fec6d877bc15e8215672e7dd8f4a4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 11:38:45 -0600 Subject: [PATCH 561/598] missing : --- salt/modules/ssh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 6df5899ea27c..79f102419c3b 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -23,7 +23,7 @@ def _refine_enc(enc): elif enc in ecdsa: # ecdsa defaults to ecdsa-sha2-nistp256 # otherwise enc string is actual encoding string - if enc in ['e', 'ecdsa'] + if enc in ['e', 'ecdsa']: return 'ecdsa-sha2-nistp256' return enc else: From 3feb8e8dabfd309ecb28a36f4c5d1950b2728ecc Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 11:42:35 -0600 Subject: [PATCH 562/598] fix typo in set creation --- salt/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index f236d4269b1a..2a2f4849395f 100644 --- a/salt/state.py +++ b/salt/state.py @@ -595,7 +595,7 @@ def requisite_in(self, high): ''' Extend the data reference with requisite_in arguments ''' - req_in = set('require_in', 'watch_in') + req_in = set(['require_in', 'watch_in']) extend = {} for id_, body in high.items(): for state, run in body.items(): From bb2a46678112857ca848616ba8c881c02ff32baf Mon Sep 17 00:00:00 2001 From: Andrew Kuhnhausen <trane@xmission.com> Date: Tue, 20 Mar 2012 11:47:40 -0600 Subject: [PATCH 563/598] Handle keys with spaces (refs #852) This patch is built on the assumption that we will at most have 4 things to add to an auth line: enc key{ keypart2} comment --- salt/modules/ssh.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 6df5899ea27c..45cd85df0814 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -23,7 +23,7 @@ def _refine_enc(enc): elif enc in ecdsa: # ecdsa defaults to ecdsa-sha2-nistp256 # otherwise enc string is actual encoding string - if enc in ['e', 'ecdsa'] + if enc in ['e', 'ecdsa']: return 'ecdsa-sha2-nistp256' return enc else: @@ -149,12 +149,21 @@ def _validate_keys(key_file): options = [] if not options: enc = comps[0] - key = comps[1] - comment = ' '.join(comps[2:]) + # check if key has a space + if len(comps) == 4: + key = comps[1] + ' ' + comps[2] + comment = ' '.join(comps[3:]) + else: + key = comps[1] + comment = ' '.join(comps[2:]) else: enc = comps[1] - key = comps[2] - comment = ' '.join(comps[3:]) + if len(comps) == 5: + key = comps[2] + ' ' + comps[3] + comment = ' '.join(comps[4:]) + else: + key = comps[2] + comment = ' '.join(comps[3:]) ret[key] = {'enc': enc, 'comment': comment, 'options': options} @@ -196,9 +205,15 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): else: options = [] if not options: - pkey = comps[1] + if len(comps) == 4: + pkey = comps[1] + ' ' + comps[2] + else: + pkey = comps[1] else: - pkey = comps[2] + if len(comps) == 4: + pkey = comps[2] + ' ' + comps[3] + else: + pkey = comps[2] if pkey == key: continue else: From 67278352b99eda998ddb244231438ab561241cc7 Mon Sep 17 00:00:00 2001 From: Andrew Kuhnhausen <trane@xmission.com> Date: Tue, 20 Mar 2012 12:05:16 -0600 Subject: [PATCH 564/598] Fix len of comps check Debugging revealed that comps array will be len 3 or 4 depending on if there are options enabled. --- salt/modules/ssh.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 45cd85df0814..5710e519e5ef 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -150,7 +150,7 @@ def _validate_keys(key_file): if not options: enc = comps[0] # check if key has a space - if len(comps) == 4: + if len(comps) == 3: key = comps[1] + ' ' + comps[2] comment = ' '.join(comps[3:]) else: @@ -158,7 +158,7 @@ def _validate_keys(key_file): comment = ' '.join(comps[2:]) else: enc = comps[1] - if len(comps) == 5: + if len(comps) == 4: key = comps[2] + ' ' + comps[3] comment = ' '.join(comps[4:]) else: @@ -205,12 +205,12 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): else: options = [] if not options: - if len(comps) == 4: + if len(comps) == 3: pkey = comps[1] + ' ' + comps[2] else: pkey = comps[1] else: - if len(comps) == 4: + if len(comps) == 3: pkey = comps[2] + ' ' + comps[3] else: pkey = comps[2] From 8417f348649b6e3c7b621b3c63794b998e9ec63d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 12:09:10 -0600 Subject: [PATCH 565/598] requisite_in statements work! --- salt/state.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 2a2f4849395f..0067461f766e 100644 --- a/salt/state.py +++ b/salt/state.py @@ -543,6 +543,8 @@ def reconcile_extend(self, high): if '__extend__' not in high: return high, errors ext = high.pop('__extend__') + import pprint + pprint.pprint(ext) for ext_chunk in ext: for name, body in ext_chunk.items(): if name not in high: @@ -647,6 +649,10 @@ def requisite_in(self, high): _state = ind.keys()[0] name = ind[_state] found = False + if not name in extend: + extend[name] = {} + if not _state in extend[name]: + extend[name][_state] = [] for ind in range(len(extend[name][_state])): if extend[name][_state][ind].keys()[0] == rkey: # Extending again @@ -660,7 +666,11 @@ def requisite_in(self, high): extend[name][_state].append( {rkey: [{state: id_}]} ) - high['__extend__'] = extend + high['__extend__'] = [] + for key, val in extend.items(): + high['__extend__'].append({key: val}) + import pprint + pprint.pprint(extend) return self.reconcile_extend(high) def call(self, data): @@ -848,6 +858,8 @@ def call_high(self, high): # If there is extension data reconcile it high, ext_errors = self.reconcile_extend(high) errors += ext_errors + high, req_in_errors = self.requisite_in(high) + errors += req_in_errors # Verify that the high data is structurally sound errors += self.verify_high(high) if errors: From 5935f42426ca18d7fcf16ed61439a43b94fc84da Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 12:10:14 -0600 Subject: [PATCH 566/598] left some print statements in there... --- salt/state.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/salt/state.py b/salt/state.py index 0067461f766e..139202a810bc 100644 --- a/salt/state.py +++ b/salt/state.py @@ -543,8 +543,6 @@ def reconcile_extend(self, high): if '__extend__' not in high: return high, errors ext = high.pop('__extend__') - import pprint - pprint.pprint(ext) for ext_chunk in ext: for name, body in ext_chunk.items(): if name not in high: @@ -669,8 +667,6 @@ def requisite_in(self, high): high['__extend__'] = [] for key, val in extend.items(): high['__extend__'].append({key: val}) - import pprint - pprint.pprint(extend) return self.reconcile_extend(high) def call(self, data): From 750af09560c641d114c3c42d3703f0afe5f6f3be Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 12:23:48 -0600 Subject: [PATCH 567/598] Change to only run reconcile_extend once --- salt/state.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/state.py b/salt/state.py index 139202a810bc..b2d82c63dc7a 100644 --- a/salt/state.py +++ b/salt/state.py @@ -598,6 +598,8 @@ def requisite_in(self, high): req_in = set(['require_in', 'watch_in']) extend = {} for id_, body in high.items(): + if not isinstance(body, dict): + continue for state, run in body.items(): if state.startswith('__'): continue @@ -664,10 +666,11 @@ def requisite_in(self, high): extend[name][_state].append( {rkey: [{state: id_}]} ) - high['__extend__'] = [] + if not '__extend__' in high: + high['__extend__'] = [] for key, val in extend.items(): high['__extend__'].append({key: val}) - return self.reconcile_extend(high) + return high def call(self, data): ''' @@ -852,10 +855,9 @@ def call_high(self, high): err = [] errors = [] # If there is extension data reconcile it + high = self.requisite_in(high) high, ext_errors = self.reconcile_extend(high) errors += ext_errors - high, req_in_errors = self.requisite_in(high) - errors += req_in_errors # Verify that the high data is structurally sound errors += self.verify_high(high) if errors: From 2d9b2ba86b93b51d9a42b3f2fa2488897cbdf9c4 Mon Sep 17 00:00:00 2001 From: Andrew Kuhnhausen <trane@xmission.com> Date: Tue, 20 Mar 2012 13:01:23 -0600 Subject: [PATCH 568/598] Refactor ssh verify line to smartly parse extra options Lines in an ssh auth key can begin like this: tunnel="1",command="sh /etc/netstart tun1" ssh-rsa Need to separate the parsing of options from the key. This patch groups the ssh auth key line into two parts, [{options}* key] --- salt/modules/ssh.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 5710e519e5ef..8e50b32044b2 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -3,6 +3,7 @@ ''' import os +import re def _refine_enc(enc): @@ -138,13 +139,16 @@ def _validate_keys(key_file): if line.startswith('#'): # Commented Line continue - comps = line.split() + ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$'); + opts = ln.group(1) + comps = ln.group(2).split() + if len(comps) < 2: # Not a valid line continue - if comps[0][:4:] not in ['ssh-', 'ecds']: + if opts: # It has options, grab them - options = comps[0].split(',') + options = opts.split(',') else: options = [] if not options: @@ -194,14 +198,17 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): # Commented Line lines.append(line) continue - comps = line.split() + ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$'); + opts = ln.group(1) + comps = ln.group(2).split() + if len(comps) < 2: # Not a valid line lines.append(line) continue - if comps[0][:4:] not in ['ssh-', 'ecds']: + if opts: # It has options, grab them - options = comps[0].split(',') + options = opts.split(',') else: options = [] if not options: From ae8d2a32915e25b973a6b3f3279039559dbebbff Mon Sep 17 00:00:00 2001 From: Andrew Kuhnhausen <trane@xmission.com> Date: Tue, 20 Mar 2012 13:05:47 -0600 Subject: [PATCH 569/598] Add second param to search() Somehow forgot this from the transition from dev branch to develop --- salt/modules/ssh.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 8e50b32044b2..42a5615e2fc0 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -139,7 +139,7 @@ def _validate_keys(key_file): if line.startswith('#'): # Commented Line continue - ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$'); + ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line); opts = ln.group(1) comps = ln.group(2).split() @@ -198,7 +198,7 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): # Commented Line lines.append(line) continue - ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$'); + ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line); opts = ln.group(1) comps = ln.group(2).split() From 15a3a124fccb076b25507c698fa5a124bd7a437f Mon Sep 17 00:00:00 2001 From: Andrew Kuhnhausen <trane@xmission.com> Date: Tue, 20 Mar 2012 13:19:36 -0600 Subject: [PATCH 570/598] Remove redudant options check Now that we are handling option parsing separately, no need for the option check --- salt/modules/ssh.py | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 42a5615e2fc0..79cddc7fccf8 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -139,6 +139,8 @@ def _validate_keys(key_file): if line.startswith('#'): # Commented Line continue + + # get "{options} key" ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line); opts = ln.group(1) comps = ln.group(2).split() @@ -151,23 +153,16 @@ def _validate_keys(key_file): options = opts.split(',') else: options = [] - if not options: - enc = comps[0] - # check if key has a space - if len(comps) == 3: - key = comps[1] + ' ' + comps[2] - comment = ' '.join(comps[3:]) - else: - key = comps[1] - comment = ' '.join(comps[2:]) + + enc = comps[0] + # check if key has a space + if len(comps) == 3: + key = comps[1] + ' ' + comps[2] + comment = ' '.join(comps[3:]) else: - enc = comps[1] - if len(comps) == 4: - key = comps[2] + ' ' + comps[3] - comment = ' '.join(comps[4:]) - else: - key = comps[2] - comment = ' '.join(comps[3:]) + key = comps[1] + comment = ' '.join(comps[2:]) + ret[key] = {'enc': enc, 'comment': comment, 'options': options} @@ -198,6 +193,8 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): # Commented Line lines.append(line) continue + + # get "{options} key" ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line); opts = ln.group(1) comps = ln.group(2).split() @@ -211,16 +208,12 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): options = opts.split(',') else: options = [] - if not options: - if len(comps) == 3: - pkey = comps[1] + ' ' + comps[2] - else: - pkey = comps[1] + + if len(comps) == 3: + pkey = comps[1] + ' ' + comps[2] else: - if len(comps) == 3: - pkey = comps[2] + ' ' + comps[3] - else: - pkey = comps[2] + pkey = comps[1] + if pkey == key: continue else: From d625e5477e3c62338e20375cd465eebcaf23c8c3 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 13:25:56 -0600 Subject: [PATCH 571/598] Add requisite in to 0.9.8 release notes --- doc/topics/releases/0.9.8.rst | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 0d36be85f5a9..4e91c3cf0353 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -15,6 +15,10 @@ assigned to minions in a granular way similar to the state system. It also allows flexibility for users who want to keep data out of their state tree similar to 'external lookup' functionality in other tools. +A new way to extend requisites was added, the "requisite in" statement. +This makes adding requires or watch statements to external stae decs +much easier. + Additions to requisites making them much more powerful have been added and improved error checking for sls files in the state system. A new provider system has been added to allow for redirecting what modules run in the @@ -179,6 +183,42 @@ More information at: https://github.com/grierj/range/wiki/Introduction-to-Range-with-YAML-files +Requisite "in" +-------------- + +A new means to updating requisite statements has been added to make adding +watchers and requires to external states easier. Before 0.9.8 the only way +to extend the states that were watched by a state outside of the sls was to +use an extend statement: + +.. code-block:: yaml + + include: + - http + extend: + apache: + service: + - watch: + - pkg: tomcat + + tomcat: + pkg: + - installed + +But the new ``Requisite in`` statement allows for easier extends for +requisites: + +.. code-block:: yaml + + include: + - http + + tomcat: + pkg: + - installed + - watch_in: + - service: apache + Providers --------- From d3aff46461c6bff957189edbbbe6e2168f7e6e0f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 13:53:01 -0600 Subject: [PATCH 572/598] Add info about running states from the minion --- doc/topics/releases/0.9.8.rst | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 4e91c3cf0353..472ac17b4d9b 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -113,6 +113,24 @@ looking through the minion logs. The *salt-key* command gained the *-D* and *--delete-all* arguments for removing all keys. Be careful with this one! +Running States Without a Master +------------------------------- + +The addition of running states without a salt-master running has been added +to 0.9.8. This feature allows for the unmodified salt state tree to be +read locally from a minion. The result is that the UNMODIFIED state tree +has just become portable, allowing minions to have a local copy of states +or to manage states without a master entirely. + +This is accomp[lished via the new file client interface in Salt that allows +for the ``salt://`` uri to be redirected to custom interfaces. This means that +there are now two interfaces for the salt file server, calling the master +or looking in a local, minion defined ``file_roots``. + +This new feature can be used by modifying the minion config to point to a +local ``file_roots`` and setting the ``file_client`` option to ``local``. + + Keyword Arguments and States ---------------------------- @@ -348,14 +366,6 @@ it to some minions without all minions having access to it. This will be good for handling ssl certificates on front-end web servers. -New File Client ---------------- - -The file client code has been re-factored to allow local ``salt://`` uris. -This will eventually allow for running salt's configuration management via -``salt-call state.highstate`` to run without a salt-minion daemon running. - - Solaris Support -------------------- From 751a9ef8ce4226043f46a3f6e2db4af2aa221c59 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 13:57:21 -0600 Subject: [PATCH 573/598] set spell in vim needs more [ power --- doc/topics/releases/0.9.8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index 472ac17b4d9b..ed52cbd59785 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -122,7 +122,7 @@ read locally from a minion. The result is that the UNMODIFIED state tree has just become portable, allowing minions to have a local copy of states or to manage states without a master entirely. -This is accomp[lished via the new file client interface in Salt that allows +This is accomplished via the new file client interface in Salt that allows for the ``salt://`` uri to be redirected to custom interfaces. This means that there are now two interfaces for the salt file server, calling the master or looking in a local, minion defined ``file_roots``. From 608151192c1bec46e3a606471faa8fe4a3fb3e93 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 14:23:49 -0600 Subject: [PATCH 574/598] Add a doc on the extend declaration --- doc/ref/states/extend.rst | 74 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 doc/ref/states/extend.rst diff --git a/doc/ref/states/extend.rst b/doc/ref/states/extend.rst new file mode 100644 index 000000000000..74c4bddeeb67 --- /dev/null +++ b/doc/ref/states/extend.rst @@ -0,0 +1,74 @@ +=========================== +Extending External SLS Data +=========================== + +Sometimes a state defined in one SLS file will need to be modified from a +separate SLS file. A good example of this is when an argument needs to be +overwritten or when a service needs to watch an additional state. + +The Extend Declaration +---------------------- + +The standard way to extend is via the extend declaration. The extend +declaration is a top level declaration like ``include`` and encapsulates ID +declaration data included from other SLS files. A standard extend looks like +this: + +.. code-block:: yaml + + include: + - http + - ssh + + extend: + apache: + file: + - name: /etc/httpd/conf/httpd.conf + - source: salt://http/httpd2.conf + ssh-server: + service: + - watch: + - file: /etc/ssh/banner + + /etc/ssh/banner: + file: + - managed + - source: salt://ssh/banner + +A few critical things happened here, first off the sls files that are going to +be extended are included, then the extend dec is defined. Under the extend dec +2 IDs are extended, the apache ID's file state is overwritten with a new name +and source. Than the ssh server is extended to watch the banner file in +addition to anything it is already watching. + +The Requisite "in" Statement +---------------------------- + +Since one of the most common things to do when extending another sls is to add +states for a service to watch, or anything for a watcher to watch, the +requisite in statement was added to 0.9.8 to make extending the watch and +require lists easier. The ssh-server extend statement above could be more +cleanly defined like so: + +.. code-block:: yaml + + include: + - ssh + + /etc/ssh/banner: + file: + - managed + - source: salt://ssh/banner + - watch_in: + - service: ssh-server + +Rules to Extend By +------------------ +There are a few rules to remember when extending states: + +1. Always include the sls being extended with an include declaration +2. Requisites (watch and require) are appended to, everything else is + overwritten +3. extend is a top level declaration, like an ID declaration, cannot be + declared twice in a single sls +4. Many IDs can be extended under the extend declaration From 44a7bf87f9613c41b0f1d6f90d6cd31c3c0410f9 Mon Sep 17 00:00:00 2001 From: archme <archme.mail@gmail.com> Date: Tue, 20 Mar 2012 21:41:36 +0100 Subject: [PATCH 575/598] 1. Make refresh_db() "locale aware" 2. Fix: ret[key] was set to "downloading" instead of <repo name> --- salt/modules/pacman.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/pacman.py b/salt/modules/pacman.py index 2f1d6fb385db..5b279336183d 100644 --- a/salt/modules/pacman.py +++ b/salt/modules/pacman.py @@ -91,7 +91,7 @@ def refresh_db(): salt '*' pkg.refresh_db ''' - cmd = 'pacman -Sy' + cmd = 'LANG=C pacman -Sy' ret = {} out = __salt__['cmd.run'](cmd).split('\n') for line in out: @@ -103,6 +103,7 @@ def refresh_db(): if 'is up to date' in line: ret[key] = False elif 'downloading' in line: + key = line.strip().split()[1].split('.')[0] ret[key] = True return ret From 6f247fe8718c12d39aa340fbd8140a0d2d6f530d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 15:05:13 -0600 Subject: [PATCH 576/598] some monor additions to the release document --- doc/topics/releases/0.9.8.rst | 39 ++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/doc/topics/releases/0.9.8.rst b/doc/topics/releases/0.9.8.rst index ed52cbd59785..77a55e1f28fb 100644 --- a/doc/topics/releases/0.9.8.rst +++ b/doc/topics/releases/0.9.8.rst @@ -16,11 +16,11 @@ allows flexibility for users who want to keep data out of their state tree similar to 'external lookup' functionality in other tools. A new way to extend requisites was added, the "requisite in" statement. -This makes adding requires or watch statements to external stae decs +This makes adding requires or watch statements to external state decs much easier. -Additions to requisites making them much more powerful have been added and -improved error checking for sls files in the state system. A new provider +Additions to requisites making them much more powerful have been added as well +as improved error checking for sls files in the state system. A new provider system has been added to allow for redirecting what modules run in the background for individual states. @@ -116,7 +116,7 @@ removing all keys. Be careful with this one! Running States Without a Master ------------------------------- -The addition of running states without a salt-master running has been added +The addition of running states without a salt-master has been added to 0.9.8. This feature allows for the unmodified salt state tree to be read locally from a minion. The result is that the UNMODIFIED state tree has just become portable, allowing minions to have a local copy of states @@ -135,7 +135,7 @@ Keyword Arguments and States ---------------------------- State modules now accept the ``**kwargs`` argument. This results in all data -in a sls file assigned to a state will be made available to the state function. +in a sls file assigned to a state being made available to the state function. This passes data in a transparent way back to the modules executing the logic. In particular, this allows adding arguments to the ``pkg.install`` module that @@ -173,11 +173,11 @@ Matcher Refinements and Changes ------------------------------- A number of fixes and changes have been applied to the Matcher system. The -most noteworthy is the change in the grain matcher. The grain matcher used -a regular expression to match the passed data to a grain, but now defaults +most noteworthy is the change in the grain matcher. The grain matcher used to +use a regular expression to match the passed data to a grain, but now defaults to a shell glob like the majority of match interfaces in Salt. A new option is available that still uses the old style regex matching to grain data called -grain-pcre. To use regex matching in compound matches use the letter *P*. +``grain-pcre``. To use regex matching in compound matches use the letter *P*. For example, this would match any ArchLinux or Fedora minions: @@ -237,6 +237,9 @@ requisites: - watch_in: - service: apache +Requisite in is part of the extend system, so still remember to always include +the sls that is being extended! + Providers --------- @@ -258,6 +261,18 @@ detected for Arch Linux, the systemd module can be used: - enable: True - provider: systemd +Default providers can also be defined in the minion config file: + +.. code-block:: yaml + + providers: + pkg: yumpkg5 + service: systemd + +When default providers are passed in the minion config, then those providers +will be applied to all functionality in Salt, this means that the functions +called by the minion will use these modules, as well as states. + Requisite Glob Matching ----------------------- @@ -354,7 +369,7 @@ Master Side State Compiling --------------------------- While we feel strongly that the advantages gained with minion side state -compiling are very critical, it does prevent certain features what may be +compiling are very critical, it does prevent certain features that may be desired. 0.9.8 has support for initial master side state compiling, but many more components still need to be developed, it is hoped that these can be finished for 0.9.9. @@ -363,18 +378,18 @@ The goal is that states can be compiled on both the master and the minion allowing for compilation to be split between master and minion. Why will this be great? It will allow storing sensitive data on the master and sending it to some minions without all minions having access to it. This will be -good for handling ssl certificates on front-end web servers. +good for handling ssl certificates on front-end web servers for instance. Solaris Support --------------------- +--------------- Salt 0.9.8 sees the introduction of basic Solaris support. The daemon runs well, but grains and more of the modules need updating and testing. Windows Support --------------------- +--------------- Salt states on windows are now much more viable thanks to contributions from our community! States for file, service, local user, and local group management are more fully From 32b235d084d3df470b3162ddf559241c500b2b08 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 16:02:45 -0600 Subject: [PATCH 577/598] there are bleeding issues with running reconcile_extend only once This change both simplifies the process and lower the number of operations and shortens the search scope for the compile --- salt/state.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/salt/state.py b/salt/state.py index b2d82c63dc7a..139202a810bc 100644 --- a/salt/state.py +++ b/salt/state.py @@ -598,8 +598,6 @@ def requisite_in(self, high): req_in = set(['require_in', 'watch_in']) extend = {} for id_, body in high.items(): - if not isinstance(body, dict): - continue for state, run in body.items(): if state.startswith('__'): continue @@ -666,11 +664,10 @@ def requisite_in(self, high): extend[name][_state].append( {rkey: [{state: id_}]} ) - if not '__extend__' in high: - high['__extend__'] = [] + high['__extend__'] = [] for key, val in extend.items(): high['__extend__'].append({key: val}) - return high + return self.reconcile_extend(high) def call(self, data): ''' @@ -855,9 +852,10 @@ def call_high(self, high): err = [] errors = [] # If there is extension data reconcile it - high = self.requisite_in(high) high, ext_errors = self.reconcile_extend(high) errors += ext_errors + high, req_in_errors = self.requisite_in(high) + errors += req_in_errors # Verify that the high data is structurally sound errors += self.verify_high(high) if errors: From f132a9abbb4bcbb34d68fee899c1210bf3562089 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 16:36:24 -0600 Subject: [PATCH 578/598] fix dict ref in requisite in --- salt/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index 139202a810bc..8d7875e6cd95 100644 --- a/salt/state.py +++ b/salt/state.py @@ -626,7 +626,7 @@ def requisite_in(self, high): for ind in range(len(extend[name][_state])): if extend[name][_state][ind].keys()[0] == rkey: # Extending again - extend[name][_state][ind].append( + extend[name][_state][ind][rkey].append( {state: id_} ) found = True @@ -654,7 +654,7 @@ def requisite_in(self, high): for ind in range(len(extend[name][_state])): if extend[name][_state][ind].keys()[0] == rkey: # Extending again - extend[name][_state][ind].append( + extend[name][_state][ind][rkey].append( {state: id_} ) found = True From ec75f1a3ae913ce79ecdb20ba8378b4eee197e57 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 16:41:06 -0600 Subject: [PATCH 579/598] don't fail out on ssh keys without options --- salt/modules/ssh.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 79cddc7fccf8..efd48f7c4d0d 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -141,9 +141,12 @@ def _validate_keys(key_file): continue # get "{options} key" - ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line); - opts = ln.group(1) - comps = ln.group(2).split() + ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line) + opts = '' + comps = '' + if ln: + opts = ln.group(1) + comps = ln.group(2).split() if len(comps) < 2: # Not a valid line From 5a2ba4508671174dd8089eebd92b84a42a544815 Mon Sep 17 00:00:00 2001 From: Andrew Kuhnhausen <trane@xmission.com> Date: Tue, 20 Mar 2012 17:28:09 -0600 Subject: [PATCH 580/598] Fix where re.search applied to empty lines Move the re.search method to ensure that lines are valid ssh key lines, break out of loop when it encounters errors. Might want to change this into exception handling of some sort. --- salt/modules/ssh.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index efd48f7c4d0d..e9b0b246f736 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -140,17 +140,18 @@ def _validate_keys(key_file): # Commented Line continue - # get "{options} key" - ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line) - opts = '' - comps = '' - if ln: - opts = ln.group(1) - comps = ln.group(2).split() - if len(comps) < 2: # Not a valid line continue + + # get "{options} key" + ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line) + if not ln: + return "Invalid SSH key" + + opts = ln.group(1) + comps = ln.group(2).split() + if opts: # It has options, grab them options = opts.split(',') @@ -197,15 +198,19 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): lines.append(line) continue - # get "{options} key" - ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line); - opts = ln.group(1) - comps = ln.group(2).split() - if len(comps) < 2: # Not a valid line lines.append(line) continue + + # get "{options} key" + ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line) + if not ln: + return "Invalid SSH key" + + opts = ln.group(1) + comps = ln.group(2).split() + if opts: # It has options, grab them options = opts.split(',') From 0c89b451e7f876c253817974f01dcbfe4b2bc687 Mon Sep 17 00:00:00 2001 From: Jens Rantil <jens.rantil@gmail.com> Date: Wed, 21 Mar 2012 00:35:50 +0100 Subject: [PATCH 581/598] Small documentation fix. "Programmer interface" => "Programming interface". --- doc/topics/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/index.rst b/doc/topics/index.rst index 7340c3fb8904..aeee15f7c344 100644 --- a/doc/topics/index.rst +++ b/doc/topics/index.rst @@ -31,7 +31,7 @@ Parallel execution The core function of Salt is to enable remote commands to be called in parallel rather than in serial, to use a secure and encrypted protocol, the smallest and -fastest network payloads possible, and with a simple programmer interface. Salt +fastest network payloads possible, and with a simple programming interface. Salt also introduces more granular controls to the realm of remote execution, allowing for commands to be executed in parallel and for systems to be targeted based on more than just hostname, but by system properties. From 90d906cfe2ddfb91e01076916f39a950d0526fb6 Mon Sep 17 00:00:00 2001 From: Andrew Kuhnhausen <trane@xmission.com> Date: Tue, 20 Mar 2012 17:40:55 -0600 Subject: [PATCH 582/598] Compile ssh key line re --- salt/modules/ssh.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index e9b0b246f736..1bdc8af75ec1 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -134,6 +134,7 @@ def _validate_keys(key_file): Return a dict containing validated keys in the passed file ''' ret = {} + linere = re.compile(r'^(.*?)\s?((?:ssh\-|ecds).+)$') try: for line in open(key_file, 'r').readlines(): if line.startswith('#'): @@ -145,7 +146,7 @@ def _validate_keys(key_file): continue # get "{options} key" - ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line) + ln = re.search(linere, line) if not ln: return "Invalid SSH key" @@ -185,6 +186,7 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): salt '*' ssh.rm_auth_key <user> <key> ''' current = auth_keys(user, config) + linere = re.compile(r'^(.*?)\s?((?:ssh\-|ecds).+)$') if key in current: # Remove the key uinfo = __salt__['user.info'](user) @@ -204,7 +206,7 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): continue # get "{options} key" - ln = re.search('(.*?)\s?((?:ssh\-|ecds).+)$', line) + ln = re.search(linere, line) if not ln: return "Invalid SSH key" From c08771d584bf29720fb1b94adf0265948362d7e9 Mon Sep 17 00:00:00 2001 From: Jens Rantil <jens.rantil@gmail.com> Date: Wed, 21 Mar 2012 00:47:06 +0100 Subject: [PATCH 583/598] Fixing a weird sentence in documentation I made a comment about the sentence here: https://github.com/saltstack/salt/commit/a7a136fd8d14644f95799ee8b1b96f0d86b320aa#commitcomment-1111231 --- doc/topics/tutorials/starting_states.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/tutorials/starting_states.rst b/doc/topics/tutorials/starting_states.rst index 64e7e280ca70..c3325a62b300 100644 --- a/doc/topics/tutorials/starting_states.rst +++ b/doc/topics/tutorials/starting_states.rst @@ -51,7 +51,7 @@ This SLS data will ensure that the package named apache is installed, and that the apache service is running. The components can be explained in a simple way. -The first like it the ID for a set of data, and it is called the ID +The first line is the ID for a set of data, and it is called the ID Declaration. This ID sets the name of the thing that needs to be manipulated. The second and fourth lines are the start of the State Declarations, so they From b4b2a0cd0a1c8ebc6f43bd3c68922333102615b5 Mon Sep 17 00:00:00 2001 From: Andrew Kuhnhausen <trane@xmission.com> Date: Tue, 20 Mar 2012 17:51:02 -0600 Subject: [PATCH 584/598] Handle empty/invalid lines by skipping This is a better way to handle empty lines or invalid authorized_keys files, just skip the offending line (just like salt was doing before the re.search patch) --- salt/modules/ssh.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 1bdc8af75ec1..685cf154b7a8 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -141,18 +141,19 @@ def _validate_keys(key_file): # Commented Line continue - if len(comps) < 2: - # Not a valid line - continue - # get "{options} key" ln = re.search(linere, line) if not ln: - return "Invalid SSH key" + # not an auth ssh key, perhaps a blank line + continue opts = ln.group(1) comps = ln.group(2).split() + if len(comps) < 2: + # Not a valid line + continue + if opts: # It has options, grab them options = opts.split(',') @@ -200,19 +201,20 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): lines.append(line) continue - if len(comps) < 2: - # Not a valid line - lines.append(line) - continue - # get "{options} key" ln = re.search(linere, line) if not ln: - return "Invalid SSH key" + # not an auth ssh key, perhaps a blank line + continue opts = ln.group(1) comps = ln.group(2).split() + if len(comps) < 2: + # Not a valid line + lines.append(line) + continue + if opts: # It has options, grab them options = opts.split(',') From 761fa13bd0ed6668466f73ac1dc8d2200f15ad63 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 19:27:32 -0600 Subject: [PATCH 585/598] Add a check to verify that the name is a string --- salt/state.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/state.py b/salt/state.py index 8d7875e6cd95..475e56addaee 100644 --- a/salt/state.py +++ b/salt/state.py @@ -330,6 +330,11 @@ def verify_high(self, high): for name, body in high.items(): if name.startswith('__'): continue + if not isinstance(name, basestring): + err = ('The name {0} in sls {1} is not formed as a ' + 'string but is a {2}').format( + name, body['__sls__'], type(name)) + errros.append(err) if not isinstance(body, dict): err = ('The type {0} in {1} is not formated as a dictionary' .format(name, body['__sls__'])) From d848d27fc9a713cdd87a1a126c73704d2f6d7247 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 19:42:08 -0600 Subject: [PATCH 586/598] Add nach check in low data verify --- salt/state.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/state.py b/salt/state.py index 475e56addaee..e2f5d105f962 100644 --- a/salt/state.py +++ b/salt/state.py @@ -257,6 +257,11 @@ def verify_data(self, data): errors.append('Missing "fun" data') if 'name' not in data: errors.append('Missing "name" data') + if not isinstance(data[name], basestring): + err = ('The name {0} in sls {1} is not formed as a ' + 'string but is a {2}').format( + name, body['__sls__'], type(name)) + errros.append(err) if errors: return errors if data['fun'].startswith('mod_'): From dbfb0296adc4c89f976229eba61688de2df83061 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 19:45:32 -0600 Subject: [PATCH 587/598] error is variable passing in errorcheck --- salt/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index e2f5d105f962..d9c08a74f088 100644 --- a/salt/state.py +++ b/salt/state.py @@ -257,10 +257,10 @@ def verify_data(self, data): errors.append('Missing "fun" data') if 'name' not in data: errors.append('Missing "name" data') - if not isinstance(data[name], basestring): + if not isinstance(data['name'], basestring): err = ('The name {0} in sls {1} is not formed as a ' 'string but is a {2}').format( - name, body['__sls__'], type(name)) + data['name'], body['__sls__'], type(data['name'])) errros.append(err) if errors: return errors From 1c5026e63b9c5e842db42efb6b8207d9ce52eca1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 20:33:31 -0600 Subject: [PATCH 588/598] Fix a name error in error checks --- salt/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index d9c08a74f088..25641fa27eef 100644 --- a/salt/state.py +++ b/salt/state.py @@ -260,7 +260,7 @@ def verify_data(self, data): if not isinstance(data['name'], basestring): err = ('The name {0} in sls {1} is not formed as a ' 'string but is a {2}').format( - data['name'], body['__sls__'], type(data['name'])) + data['name'], data['__sls__'], type(data['name'])) errros.append(err) if errors: return errors From bbd94b66c22560f94d673735b3cb14d3b3dc0f0b Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 20:35:45 -0600 Subject: [PATCH 589/598] not my day... --- salt/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 25641fa27eef..c1a5eddf8b07 100644 --- a/salt/state.py +++ b/salt/state.py @@ -261,7 +261,7 @@ def verify_data(self, data): err = ('The name {0} in sls {1} is not formed as a ' 'string but is a {2}').format( data['name'], data['__sls__'], type(data['name'])) - errros.append(err) + erros.append(err) if errors: return errors if data['fun'].startswith('mod_'): From 3e40e8702b32dd1d97001160f48f62750562b14a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch <thatch45@gmail.com> Date: Tue, 20 Mar 2012 20:37:51 -0600 Subject: [PATCH 590/598] miss spelled errors in in verify high --- salt/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index c1a5eddf8b07..307381d13fff 100644 --- a/salt/state.py +++ b/salt/state.py @@ -261,7 +261,7 @@ def verify_data(self, data): err = ('The name {0} in sls {1} is not formed as a ' 'string but is a {2}').format( data['name'], data['__sls__'], type(data['name'])) - erros.append(err) + errors.append(err) if errors: return errors if data['fun'].startswith('mod_'): @@ -339,7 +339,7 @@ def verify_high(self, high): err = ('The name {0} in sls {1} is not formed as a ' 'string but is a {2}').format( name, body['__sls__'], type(name)) - errros.append(err) + errors.append(err) if not isinstance(body, dict): err = ('The type {0} in {1} is not formated as a dictionary' .format(name, body['__sls__'])) From 32e696c6ad9110593b43f4a4e16326581f5a3947 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick@nicklang.com> Date: Tue, 20 Mar 2012 22:27:05 -0600 Subject: [PATCH 591/598] adding check to see if psql bin exist, and only reutn the module if it does --- salt/modules/postgres.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index 4ade43455ee7..a26758c4d518 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -14,11 +14,25 @@ ''' import logging +from salt.utils import check_or_die +from salt.exceptions import CommandNotFoundError + log = logging.getLogger(__name__) __opts__ = {} +def __virtual__(): + """ + only load this module if the psql bin exists + """ + try: + check_or_die('psql') + return 'postgres' + except CommandNotFoundError: + return False + + def version(): ''' Return the version of a Postgres server using the output From ca4d032e0157f66d3dd2c1eb8f92298ff9e1d742 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick@nicklang.com> Date: Tue, 20 Mar 2012 23:47:02 -0600 Subject: [PATCH 592/598] adding django module --- salt/modules/django.py | 112 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 salt/modules/django.py diff --git a/salt/modules/django.py b/salt/modules/django.py new file mode 100644 index 000000000000..e003082fae88 --- /dev/null +++ b/salt/modules/django.py @@ -0,0 +1,112 @@ + + +def _get_django_admin(bin_env): + if not bin_env: + da = 'django-admin.py' + else: + # try to get pip bin from env + if os.path.exists(os.path.join(bin_env, 'bin', 'django-admin.py')): + da = os.path.join(bin_env, 'bin', 'django-admin.py') + else: + da = bin_env + return da + + +def command(settings_module, command, bin_env=None, *args, **kwargs): + """ + run arbitrary django management command + """ + da = _get_django_admin(bin_env) + cmd = "{0} {1} --settings={2}".format(da, command, settings_module) + + for arg in args: + cmd = "{0} --{1}".format(cmd, arg) + + for key, value in kwargs.iteritems(): + cmd = '{0} --{1}={2}'.format(cmd, key, value) + + return __salt__['cmd.run'](cmd) + + +def syncdb(settings_module, bin_env=None, migrate=False, database=None): + """ + run syncdb + + if you have south installed, you can pass in the optional + ``migrate`` kwarg and run the migrations after the syncdb + finishes. + """ + da = _get_django_admin(bin_env) + cmd = "{0} syncdb --settings={1}".format(da, settings_module) + if migrate: + cmd = "{0} --migrate".format(cmd) + if database: + cmd = "{0} --database={1}".format(cmd, database) + return __salt__['cmd.run'](cmd) + + +def createsuperuser(settings_module, + username, + email, + bin_env=None, + database=None): + """ + create a super user for the database. + this defaults to use the ``--noinput`` flag which will + not create a password for the superuser. + """ + da = _get_django_admin(bin_env) + cmd = "{0} createsuperuser --settings={1} --noinput --email={2} --username={3}".format( + da, settings_module, username, email) + if database: + cmd = "{0} --database={1}".format(cmd, database) + return __salt__['cmd.run'](cmd) + + +def loaddata(settings_module, fixtures, bin_env=None, database=None): + """ + load fixture data + + fixtures: + comma separated list of fixtures to load + + """ + da = _get_django_admin(bin_env) + cmd = "{0} loaddata --settings={1} {2}".format( + da, settings_module, " ".join(fixtures.split(","))) + if database: + cmd = "{0} --database={1}".format(cmd, database) + return __salt__['cmd.run'](cmd) + + +def collectstatic(settings_module, + bin_env=None, + no_post_process=False, + ignore=None, + dry_run=False, + clear=False, + link=False, + no_default_ignore=False): + + da = _get_django_admin(bin_env) + cmd = "{0} collectstatic --settings={1} --noinput".format( + da, settings_module) + if no_post_process: + cmd = "{0} --no-post-process".format(cmd) + if ignore: + cmd = "{0} --ignore=".format(cmd, ignore) + if dry_run: + cmd = "{0} --dry-run".format(cmd) + if clear: + cmd = "{0} --clear".format(cmd) + if link: + cmd = "{0} --link".format(cmd) + if no_default_ignore: + cmd = "{0} --no-default-ignore".format(cmd) + + return __salt__['cmd.run'](cmd) + + + + + From b492f38c9f46b687738e5debb0ac7c42cc714804 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Wed, 21 Mar 2012 06:03:26 +0000 Subject: [PATCH 593/598] create user function quotes were messed up, working now --- salt/modules/postgres.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index a26758c4d518..9c8e3f44abcb 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -222,10 +222,10 @@ def user_create(username, if sub_cmd.endswith("WITH"): sub_cmd = sub_cmd.replace(" WITH", "") - - cmd = "psql -h {host} -U {user} -p {port} -c '{sub_cmd}'".format( + + cmd = 'psql -h {host} -U {user} -p {port} -c "{sub_cmd}"'.format( host=host, user=user, port=port, sub_cmd=sub_cmd) - + print cmd return __salt__['cmd.run'](cmd) From fb218b7462a830584aa429b160c4d6834d96ce3f Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Wed, 21 Mar 2012 06:18:28 +0000 Subject: [PATCH 594/598] adding python path flag --- salt/modules/django.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/modules/django.py b/salt/modules/django.py index e003082fae88..f8a4cb5fe6b7 100644 --- a/salt/modules/django.py +++ b/salt/modules/django.py @@ -1,4 +1,4 @@ - +import os def _get_django_admin(bin_env): if not bin_env: @@ -28,7 +28,7 @@ def command(settings_module, command, bin_env=None, *args, **kwargs): return __salt__['cmd.run'](cmd) -def syncdb(settings_module, bin_env=None, migrate=False, database=None): +def syncdb(settings_module, bin_env=None, migrate=False, database=None, pythonpath=None): """ run syncdb @@ -42,6 +42,9 @@ def syncdb(settings_module, bin_env=None, migrate=False, database=None): cmd = "{0} --migrate".format(cmd) if database: cmd = "{0} --database={1}".format(cmd, database) + if pythonpath: + cmd = "{0} --pythonpath={1}".format(cmd, pythonpath) + print cmd return __salt__['cmd.run'](cmd) From 86a57ba69b65fe8978a50102d2c92cf9b620e556 Mon Sep 17 00:00:00 2001 From: Nick Lang <nick@nicklang.com> Date: Wed, 21 Mar 2012 00:19:59 -0600 Subject: [PATCH 595/598] adding python path flag to the rest of the bunch --- salt/modules/django.py | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/salt/modules/django.py b/salt/modules/django.py index f8a4cb5fe6b7..cca698163463 100644 --- a/salt/modules/django.py +++ b/salt/modules/django.py @@ -12,13 +12,20 @@ def _get_django_admin(bin_env): return da -def command(settings_module, command, bin_env=None, *args, **kwargs): +def command(settings_module, + command, + bin_env=None, + pythonpath=None, + *args, **kwargs): """ run arbitrary django management command """ da = _get_django_admin(bin_env) cmd = "{0} {1} --settings={2}".format(da, command, settings_module) + if pythonpath: + cmd = "{0} --pythonpath={1}".format(cmd, pythonpath) + for arg in args: cmd = "{0} --{1}".format(cmd, arg) @@ -28,7 +35,11 @@ def command(settings_module, command, bin_env=None, *args, **kwargs): return __salt__['cmd.run'](cmd) -def syncdb(settings_module, bin_env=None, migrate=False, database=None, pythonpath=None): +def syncdb(settings_module, + bin_env=None, + migrate=False, + database=None, + pythonpath=None): """ run syncdb @@ -52,7 +63,8 @@ def createsuperuser(settings_module, username, email, bin_env=None, - database=None): + database=None, + pythonpath=None): """ create a super user for the database. this defaults to use the ``--noinput`` flag which will @@ -63,10 +75,16 @@ def createsuperuser(settings_module, da, settings_module, username, email) if database: cmd = "{0} --database={1}".format(cmd, database) + if pythonpath: + cmd = "{0} --pythonpath={1}".format(cmd, pythonpath) return __salt__['cmd.run'](cmd) -def loaddata(settings_module, fixtures, bin_env=None, database=None): +def loaddata(settings_module, + fixtures, + bin_env=None, + database=None, + pythonpath=None): """ load fixture data @@ -79,6 +97,8 @@ def loaddata(settings_module, fixtures, bin_env=None, database=None): da, settings_module, " ".join(fixtures.split(","))) if database: cmd = "{0} --database={1}".format(cmd, database) + if pythonpath: + cmd = "{0} --pythonpath={1}".format(cmd, pythonpath) return __salt__['cmd.run'](cmd) @@ -89,7 +109,8 @@ def collectstatic(settings_module, dry_run=False, clear=False, link=False, - no_default_ignore=False): + no_default_ignore=False, + pythonpath=None): da = _get_django_admin(bin_env) cmd = "{0} collectstatic --settings={1} --noinput".format( @@ -106,10 +127,7 @@ def collectstatic(settings_module, cmd = "{0} --link".format(cmd) if no_default_ignore: cmd = "{0} --no-default-ignore".format(cmd) + if pythonpath: + cmd = "{0} --pythonpath={1}".format(cmd, pythonpath) return __salt__['cmd.run'](cmd) - - - - - From a852b5289454f31327ce983060b4663dbfbe14b8 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Wed, 21 Mar 2012 06:27:27 +0000 Subject: [PATCH 596/598] fixing order of username/email for command --- salt/modules/django.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/django.py b/salt/modules/django.py index cca698163463..4b6f900c6bbc 100644 --- a/salt/modules/django.py +++ b/salt/modules/django.py @@ -71,8 +71,8 @@ def createsuperuser(settings_module, not create a password for the superuser. """ da = _get_django_admin(bin_env) - cmd = "{0} createsuperuser --settings={1} --noinput --email={2} --username={3}".format( - da, settings_module, username, email) + cmd = "{0} createsuperuser --settings={1} --noinput --email='{2}' --username={3}".format( + da, settings_module, email, username) if database: cmd = "{0} --database={1}".format(cmd, database) if pythonpath: From 1ea9bc419741d5d4310246203b60a77fb89e6f67 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Wed, 21 Mar 2012 06:28:38 +0000 Subject: [PATCH 597/598] removing print statement --- salt/modules/postgres.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index 9c8e3f44abcb..90881294374c 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -225,7 +225,6 @@ def user_create(username, cmd = 'psql -h {host} -U {user} -p {port} -c "{sub_cmd}"'.format( host=host, user=user, port=port, sub_cmd=sub_cmd) - print cmd return __salt__['cmd.run'](cmd) From 5162e39ad53ffbabac07df9485d8788f0472fbd7 Mon Sep 17 00:00:00 2001 From: "Nick Lang (Salt minion)" <nick.lang@gmail.com> Date: Wed, 21 Mar 2012 06:29:17 +0000 Subject: [PATCH 598/598] removing print statement --- salt/modules/django.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/modules/django.py b/salt/modules/django.py index 4b6f900c6bbc..a0dd40502a3b 100644 --- a/salt/modules/django.py +++ b/salt/modules/django.py @@ -55,7 +55,6 @@ def syncdb(settings_module, cmd = "{0} --database={1}".format(cmd, database) if pythonpath: cmd = "{0} --pythonpath={1}".format(cmd, pythonpath) - print cmd return __salt__['cmd.run'](cmd)