From 55115ff122dbab80afdd8036fb9dfb4bcd9adeae Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Wed, 13 Mar 2024 18:10:09 +0000 Subject: [PATCH 1/3] feat(log): extra logging for local ckan; - Log execution of ckanapi commands for local ckan. --- README.md | 46 +++++++++++++++++++++++++++++++++++++++++++-- ckanapi/cli/main.py | 25 ++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4e5858f..e41242c 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ multiple worker processes using the `-p` parameter. The jobs in progress, the rate of job completion and any individual errors are shown on STDERR while the jobs run. -There are no parallel limits when running against a CKAN on localhost. +There are no parallel limits when running against a CKAN on localhost. When running against a remote site, there's a default limit of 3 worker processes. The environment variables `CKANAPI_MY_SITES` and`CKANAPI_PARALLEL_LIMIT` can be @@ -352,7 +352,7 @@ ua = 'ckanapiexample/1.0 (+http://example.com/my/website)' demo = RemoteCKAN('https://demo.ckan.org', user_agent=ua) packages = demo.action.package_search(q='+organization:sample-organization +res_format:GeoJSON +tags:geojson') print(packages) -``` +``` Many CKAN API functions can only be used by authenticated users. Use the `apikey` parameter to supply your CKAN API key to `RemoteCKAN`: @@ -463,6 +463,48 @@ anon = LocalCKAN(username='') print(anon.action.status_show()) ``` +#### Extra Loggging + +To enable extra debug logging for the execution of LocalCKAN ckanapi commands, you can enable the config option in your CKAN INI file. + +``` +ckanapi.log_local = True +``` + +The output of the log will look like: + +``` +DEBUG [ckanapi.cli.main] OS User executed LocalCKAN: ckanapi +``` + +Because the ckanapi namespace does not match ckan or ckanext namespaces, you may need to create a new logging handler. Example: + +``` +[loggers] +keys = ckanapi + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_ckanapi] +level = DEBUG +handlers = console +qualname = ckanapi +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s +``` + ### TestAppCKAN A class is provided for making action requests to a diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py index 6f37a28..3b88654 100644 --- a/ckanapi/cli/main.py +++ b/ckanapi/cli/main.py @@ -86,6 +86,7 @@ import os from docopt import docopt from pkg_resources import load_entry_point +import subprocess from ckanapi.version import __version__ from ckanapi.remoteckan import RemoteCKAN @@ -98,7 +99,10 @@ from ckanapi.cli.search import search_datasets from ckanapi.cli.batch import batch_actions +from logging import getLogger + +log = getLogger(__name__) PYTHON2 = str is bytes def parse_arguments(): @@ -130,6 +134,27 @@ def main(running_with_paster=False): ) else: ckan = LocalCKAN(username=arguments['--ckan-user']) + # log execution of LocalCKAN commands + from ckan.plugins.toolkit import config, asbool + if asbool(config.get('ckanapi.log_local')) and len(sys.argv) > 1: + cmd = ['who', 'am', 'i'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = proc.communicate() + if not out or err: + # fallback to whoami if `who am i` is empty or errored + cmd = ['whoami'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = proc.communicate() + if not out or err: + # cannot find user + out = '' + else: + # remove line breaks from whoami's + out = out.replace('\n', '').replace('\r', '') + log.debug('OS User %s executed LocalCKAN: ckanapi %s', + out, u' '.join(sys.argv[1:])) stdout = getattr(sys.stdout, 'buffer', sys.stdout) if arguments['action']: From 3363e04bcc59dcc49ba44aef0ead23f7fc3120ce Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Wed, 13 Mar 2024 18:34:29 +0000 Subject: [PATCH 2/3] feat(log): logging namespace, whoami split; - Use explicit `ckan.ckanapi` logging name for easier logging handlers. - Added `split` for the `who am i` output. - Changed logging level from `debug` to `info`. --- README.md | 30 +----------------------------- ckanapi/cli/main.py | 10 ++++++---- 2 files changed, 7 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index e41242c..8e03cdb 100644 --- a/README.md +++ b/README.md @@ -474,35 +474,7 @@ ckanapi.log_local = True The output of the log will look like: ``` -DEBUG [ckanapi.cli.main] OS User executed LocalCKAN: ckanapi -``` - -Because the ckanapi namespace does not match ckan or ckanext namespaces, you may need to create a new logging handler. Example: - -``` -[loggers] -keys = ckanapi - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_ckanapi] -level = DEBUG -handlers = console -qualname = ckanapi -propagate = 0 - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s +DEBUG [ckan.ckanapi] OS User executed LocalCKAN: ckanapi ``` ### TestAppCKAN diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py index 3b88654..a69e35b 100644 --- a/ckanapi/cli/main.py +++ b/ckanapi/cli/main.py @@ -101,8 +101,8 @@ from logging import getLogger - -log = getLogger(__name__) +# explicit logger namespace for easy logging handlers +log = getLogger('ckan.ckanapi') PYTHON2 = str is bytes def parse_arguments(): @@ -153,8 +153,10 @@ def main(running_with_paster=False): else: # remove line breaks from whoami's out = out.replace('\n', '').replace('\r', '') - log.debug('OS User %s executed LocalCKAN: ckanapi %s', - out, u' '.join(sys.argv[1:])) + # split the `who am i` + out = out.split()[0] + log.info('OS User %s executed LocalCKAN: ckanapi %s', + out, u' '.join(sys.argv[1:])) stdout = getattr(sys.stdout, 'buffer', sys.stdout) if arguments['action']: From a6f98bc1d0828f558454aa12f7e6bb4efe0bfbb3 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Wed, 13 Mar 2024 18:38:13 +0000 Subject: [PATCH 3/3] feat(readme): log level in readm; - Changed logging level from `debug` to `info`. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e03cdb..3d2f003 100644 --- a/README.md +++ b/README.md @@ -465,7 +465,7 @@ print(anon.action.status_show()) #### Extra Loggging -To enable extra debug logging for the execution of LocalCKAN ckanapi commands, you can enable the config option in your CKAN INI file. +To enable extra info logging for the execution of LocalCKAN ckanapi commands, you can enable the config option in your CKAN INI file. ``` ckanapi.log_local = True @@ -474,7 +474,7 @@ ckanapi.log_local = True The output of the log will look like: ``` -DEBUG [ckan.ckanapi] OS User executed LocalCKAN: ckanapi +INFO [ckan.ckanapi] OS User executed LocalCKAN: ckanapi ``` ### TestAppCKAN