From ec730aef563280b1f295359f7d6cd1150df5a0f4 Mon Sep 17 00:00:00 2001 From: renoyjohnm <168143499+renoyjohnm@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:09:51 -0800 Subject: [PATCH 01/17] Fix for not retrieving all items when response has multiple pages of items (#318) --- tabcmd/commands/server.py | 53 ++++-- tabcmd/execution/parent_parser.py | 8 + tabcmd/execution/tabcmd_controller.py | 3 + tests/commands/test_projects_utils.py | 6 +- tests/commands/test_publish_command.py | 7 +- tests/commands/test_run_commands.py | 9 +- tests/commands/test_server.py | 222 +++++++++++++++++++++++++ 7 files changed, 288 insertions(+), 20 deletions(-) create mode 100644 tests/commands/test_server.py diff --git a/tabcmd/commands/server.py b/tabcmd/commands/server.py index 4091c0ba..46e77d01 100644 --- a/tabcmd/commands/server.py +++ b/tabcmd/commands/server.py @@ -50,31 +50,52 @@ def get_items_by_name(logger, item_endpoint, item_name: str, container: Optional container_name: str = "[{0}] {1}".format(container.__class__.__name__, container.name) item_log_name = "{0}/{1}".format(container_name, item_log_name) logger.debug(_("export.status").format(item_log_name)) - req_option = TSC.RequestOptions() - req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name)) - all_items, pagination_item = item_endpoint.get(req_option) - if all_items is None or all_items == []: - raise TSC.ServerResponseError( - code="404", - summary=_("errors.xmlapi.not_found"), - detail=_("errors.xmlapi.not_found") + ": " + item_log_name, + + result = [] + total_available_items = None + page_number = 1 + total_retrieved_items = 0 + + while True: + req_option = TSC.RequestOptions(pagenumber=page_number) + req_option.filter.add( + TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name) ) - if len(all_items) == 1: - logger.debug("Exactly one result found") - result = all_items - if len(all_items) > 1: + all_items, pagination_item = item_endpoint.get(req_option) + + if all_items is None or all_items == []: + raise TSC.ServerResponseError( + code="404", + summary=_("errors.xmlapi.not_found"), + detail=_("errors.xmlapi.not_found") + ": " + item_log_name, + ) + + if total_available_items is None: + total_available_items = pagination_item.total_available + + total_retrieved_items += len(all_items) + logger.debug( - "{}+ items of this name were found: {}".format( - len(all_items), all_items[0].name + ", " + all_items[1].name + ", ..." + "{} items of name: {} were found for query page number: {}, page size: {} & total available: {}".format( + len(all_items), + item_name, + pagination_item.page_number, + pagination_item.page_size, + pagination_item.total_available, ) ) if container: container_id = container.id logger.debug("Filtering to items in project {}".format(container.id)) - result = list(filter(lambda item: item.project_id == container_id, all_items)) + result.extend(list(filter(lambda item: item.project_id == container_id, all_items))) else: - result = all_items + result.extend(all_items) + + if total_retrieved_items >= total_available_items: + break + + page_number = pagination_item.page_number + 1 return result diff --git a/tabcmd/execution/parent_parser.py b/tabcmd/execution/parent_parser.py index bc81f59c..41362bed 100644 --- a/tabcmd/execution/parent_parser.py +++ b/tabcmd/execution/parent_parser.py @@ -134,6 +134,14 @@ def parent_parser_with_global_options(): version=strings[6] + "v" + version + "\n \n", help=strings[7], ) + + parser.add_argument( + "--query-page-size", + type=int, + default=None, + metavar="", + help="Specify the page size for query results.", + ) return parser diff --git a/tabcmd/execution/tabcmd_controller.py b/tabcmd/execution/tabcmd_controller.py index 6e06dfe1..2fbf270d 100644 --- a/tabcmd/execution/tabcmd_controller.py +++ b/tabcmd/execution/tabcmd_controller.py @@ -1,4 +1,5 @@ import logging +import os import sys from .localize import set_client_locale @@ -37,6 +38,8 @@ def run(parser, user_input=None): logger.debug(namespace) if namespace.language: set_client_locale(namespace.language, logger) + if namespace.query_page_size: + os.environ["TSC_PAGE_SIZE"] = str(namespace.query_page_size) try: func = namespace.func # if a subcommand was identified, call the function assigned to it diff --git a/tests/commands/test_projects_utils.py b/tests/commands/test_projects_utils.py index 9ba3c09d..5e92a10d 100644 --- a/tests/commands/test_projects_utils.py +++ b/tests/commands/test_projects_utils.py @@ -7,7 +7,11 @@ fake_item = mock.MagicMock() fake_item.name = "fake-name" fake_item.id = "fake-id" -getter = mock.MagicMock("get", return_value=([fake_item], 1)) +fake_item_pagination = mock.MagicMock() +fake_item_pagination.page_number = 1 +fake_item_pagination.total_available = 1 +fake_item_pagination.page_size = 100 +getter = mock.MagicMock("get", return_value=([fake_item], fake_item_pagination)) class ProjectsTest(unittest.TestCase): diff --git a/tests/commands/test_publish_command.py b/tests/commands/test_publish_command.py index 0c9049a7..800c81fc 100644 --- a/tests/commands/test_publish_command.py +++ b/tests/commands/test_publish_command.py @@ -18,12 +18,17 @@ fake_item.pdf = b"/pdf-representation-of-view" fake_item.extract_encryption_mode = "Disabled" +fake_item_pagination = MagicMock() +fake_item_pagination.page_number = 1 +fake_item_pagination.total_available = 1 +fake_item_pagination.page_size = 100 + fake_job = MagicMock() fake_job.id = "fake-job-id" creator = MagicMock() getter = MagicMock() -getter.get = MagicMock("get", return_value=([fake_item], 1)) +getter.get = MagicMock("get", return_value=([fake_item], fake_item_pagination)) getter.publish = MagicMock("publish", return_value=fake_item) diff --git a/tests/commands/test_run_commands.py b/tests/commands/test_run_commands.py index c29341d0..c24e3e74 100644 --- a/tests/commands/test_run_commands.py +++ b/tests/commands/test_run_commands.py @@ -37,15 +37,21 @@ fake_item = MagicMock() fake_item.name = "fake-name" fake_item.id = "fake-id" +fake_item.project_id = "fake-id" fake_item.pdf = b"/pdf-representation-of-view" fake_item.extract_encryption_mode = "Disabled" +fake_item_pagination = MagicMock() +fake_item_pagination.page_number = 1 +fake_item_pagination.total_available = 1 +fake_item_pagination.page_size = 100 + fake_job = MagicMock() fake_job.id = "fake-job-id" creator = MagicMock() getter = MagicMock() -getter.get = MagicMock("get", return_value=([fake_item], 1)) +getter.get = MagicMock("get", return_value=([fake_item], fake_item_pagination)) getter.publish = MagicMock("publish", return_value=fake_item) getter.create_extract = MagicMock("create_extract", return_value=fake_job) getter.decrypt_extract = MagicMock("decrypt_extract", return_value=fake_job) @@ -214,7 +220,6 @@ def test_refresh_extract(self, mock_session, mock_server): mock_args.removecalculations = None mock_args.incremental = None mock_args.synchronous = None - print(mock_args) refresh_extracts_command.RefreshExtracts.run_command(mock_args) mock_session.assert_called() diff --git a/tests/commands/test_server.py b/tests/commands/test_server.py new file mode 100644 index 00000000..b8a7d293 --- /dev/null +++ b/tests/commands/test_server.py @@ -0,0 +1,222 @@ +import unittest +from unittest.mock import MagicMock, patch +import tableauserverclient as TSC +from tabcmd.commands.server import Server + + +class TestServer(unittest.TestCase): + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_returns_items(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = None + + pagination_item = MagicMock() + pagination_item.total_available = 1 + pagination_item.page_number = 1 + pagination_item.page_size = 1 + + item = MagicMock() + item_endpoint.get.return_value = ([item], pagination_item) + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, [item]) + logger.debug.assert_called() + item_endpoint.get.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_no_items_found(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = None + + pagination_item = MagicMock() + pagination_item.total_available = 0 + pagination_item.page_number = 1 + pagination_item.page_size = 1 + + item_endpoint.get.return_value = ([], pagination_item) + + with self.assertRaises(TSC.ServerResponseError): + Server.get_items_by_name(logger, item_endpoint, item_name, container) + + logger.debug.assert_called() + item_endpoint.get.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_with_container(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = MagicMock() + container.id = "container_id" + + pagination_item = MagicMock() + pagination_item.total_available = 1 + pagination_item.page_number = 1 + pagination_item.page_size = 1 + + item = MagicMock() + item.project_id = "container_id" + item_endpoint.get.return_value = ([item], pagination_item) + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, [item]) + logger.debug.assert_called() + item_endpoint.get.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_with_container_no_match(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = MagicMock() + container.id = "container_id" + + pagination_item = MagicMock() + pagination_item.total_available = 1 + pagination_item.page_number = 1 + pagination_item.page_size = 1 + + item = MagicMock() + item.project_id = "different_container_id" + item_endpoint.get.return_value = ([item], pagination_item) + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, []) + logger.debug.assert_called() + item_endpoint.get.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_multiple_pages(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = None + + pagination_item_1 = MagicMock() + pagination_item_1.total_available = 3 + pagination_item_1.page_number = 1 + pagination_item_1.page_size = 1 + + pagination_item_2 = MagicMock() + pagination_item_2.total_available = 3 + pagination_item_2.page_number = 2 + pagination_item_2.page_size = 1 + + pagination_item_3 = MagicMock() + pagination_item_3.total_available = 3 + pagination_item_3.page_number = 3 + pagination_item_3.page_size = 1 + + item_1 = MagicMock() + item_2 = MagicMock() + item_3 = MagicMock() + + item_endpoint.get.side_effect = [ + ([item_1], pagination_item_1), + ([item_2], pagination_item_2), + ([item_3], pagination_item_3), + ] + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, [item_1, item_2, item_3]) + self.assertEqual(item_endpoint.get.call_count, 3) + logger.debug.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_multiple_pages_with_container(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = MagicMock() + container.id = "container_id" + + pagination_item_1 = MagicMock() + pagination_item_1.total_available = 3 + pagination_item_1.page_number = 1 + pagination_item_1.page_size = 1 + + pagination_item_2 = MagicMock() + pagination_item_2.total_available = 3 + pagination_item_2.page_number = 2 + pagination_item_2.page_size = 1 + + pagination_item_3 = MagicMock() + pagination_item_3.total_available = 3 + pagination_item_3.page_number = 3 + pagination_item_3.page_size = 1 + + item_1 = MagicMock() + item_1.project_id = "container_id_1" + item_2 = MagicMock() + item_2.project_id = "container_id" + item_3 = MagicMock() + item_3.project_id = "container_id_2" + + item_endpoint.get.side_effect = [ + ([item_1], pagination_item_1), + ([item_2], pagination_item_2), + ([item_3], pagination_item_3), + ] + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, [item_2]) + self.assertEqual(item_endpoint.get.call_count, 3) + logger.debug.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_multiple_pages_no_container_match(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = MagicMock() + container.id = "container_id" + + pagination_item_1 = MagicMock() + pagination_item_1.total_available = 3 + pagination_item_1.page_number = 1 + pagination_item_1.page_size = 1 + + pagination_item_2 = MagicMock() + pagination_item_2.total_available = 3 + pagination_item_2.page_number = 2 + pagination_item_2.page_size = 1 + + pagination_item_3 = MagicMock() + pagination_item_3.total_available = 3 + pagination_item_3.page_number = 3 + pagination_item_3.page_size = 1 + + item_1 = MagicMock() + item_1.project_id = "different_container_id_1" + item_2 = MagicMock() + item_2.project_id = "different_container_id_2" + item_3 = MagicMock() + item_3.project_id = "different_container_id_3" + + item_endpoint.get.side_effect = [ + ([item_1], pagination_item_1), + ([item_2], pagination_item_2), + ([item_3], pagination_item_3), + ] + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, []) + self.assertEqual(item_endpoint.get.call_count, 3) + logger.debug.assert_called() From 0075f1788ddd968c00f439c083c9926cfbd80dfe Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Sat, 21 Dec 2024 00:16:17 -0800 Subject: [PATCH 02/17] fix for error when item is not in first page I pushed the parent-project filter into the request so we only get the results that match. We should rename this method [and others] to make it clear they will return one item or throw an error. --- tabcmd/commands/server.py | 22 ++++--- tests/commands/test_server.py | 110 +--------------------------------- 2 files changed, 11 insertions(+), 121 deletions(-) diff --git a/tabcmd/commands/server.py b/tabcmd/commands/server.py index 46e77d01..018e9737 100644 --- a/tabcmd/commands/server.py +++ b/tabcmd/commands/server.py @@ -61,18 +61,22 @@ def get_items_by_name(logger, item_endpoint, item_name: str, container: Optional req_option.filter.add( TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name) ) + if container: + req_option.filter.add( + TSC.Filter( + TSC.RequestOptions.Field.ParentProjectId, TSC.RequestOptions.Operator.Equals, container.id + ) + ) + all_items, pagination_item = item_endpoint.get(req_option) - if all_items is None or all_items == []: + if pagination_item.total_available == 0: raise TSC.ServerResponseError( code="404", summary=_("errors.xmlapi.not_found"), detail=_("errors.xmlapi.not_found") + ": " + item_log_name, ) - if total_available_items is None: - total_available_items = pagination_item.total_available - total_retrieved_items += len(all_items) logger.debug( @@ -85,14 +89,8 @@ def get_items_by_name(logger, item_endpoint, item_name: str, container: Optional ) ) - if container: - container_id = container.id - logger.debug("Filtering to items in project {}".format(container.id)) - result.extend(list(filter(lambda item: item.project_id == container_id, all_items))) - else: - result.extend(all_items) - - if total_retrieved_items >= total_available_items: + result.extend(all_items) + if total_retrieved_items >= pagination_item.total_available: break page_number = pagination_item.page_number + 1 diff --git a/tests/commands/test_server.py b/tests/commands/test_server.py index b8a7d293..6f8ed058 100644 --- a/tests/commands/test_server.py +++ b/tests/commands/test_server.py @@ -1,3 +1,4 @@ +import pytest import unittest from unittest.mock import MagicMock, patch import tableauserverclient as TSC @@ -72,29 +73,6 @@ def test_get_items_by_name_with_container(self, MockFilter, MockRequestOptions): logger.debug.assert_called() item_endpoint.get.assert_called() - @patch("tabcmd.commands.server.TSC.RequestOptions") - @patch("tabcmd.commands.server.TSC.Filter") - def test_get_items_by_name_with_container_no_match(self, MockFilter, MockRequestOptions): - logger = MagicMock() - item_endpoint = MagicMock() - item_name = "test_item" - container = MagicMock() - container.id = "container_id" - - pagination_item = MagicMock() - pagination_item.total_available = 1 - pagination_item.page_number = 1 - pagination_item.page_size = 1 - - item = MagicMock() - item.project_id = "different_container_id" - item_endpoint.get.return_value = ([item], pagination_item) - - result = Server.get_items_by_name(logger, item_endpoint, item_name, container) - - self.assertEqual(result, []) - logger.debug.assert_called() - item_endpoint.get.assert_called() @patch("tabcmd.commands.server.TSC.RequestOptions") @patch("tabcmd.commands.server.TSC.Filter") @@ -134,89 +112,3 @@ def test_get_items_by_name_multiple_pages(self, MockFilter, MockRequestOptions): self.assertEqual(result, [item_1, item_2, item_3]) self.assertEqual(item_endpoint.get.call_count, 3) logger.debug.assert_called() - - @patch("tabcmd.commands.server.TSC.RequestOptions") - @patch("tabcmd.commands.server.TSC.Filter") - def test_get_items_by_name_multiple_pages_with_container(self, MockFilter, MockRequestOptions): - logger = MagicMock() - item_endpoint = MagicMock() - item_name = "test_item" - container = MagicMock() - container.id = "container_id" - - pagination_item_1 = MagicMock() - pagination_item_1.total_available = 3 - pagination_item_1.page_number = 1 - pagination_item_1.page_size = 1 - - pagination_item_2 = MagicMock() - pagination_item_2.total_available = 3 - pagination_item_2.page_number = 2 - pagination_item_2.page_size = 1 - - pagination_item_3 = MagicMock() - pagination_item_3.total_available = 3 - pagination_item_3.page_number = 3 - pagination_item_3.page_size = 1 - - item_1 = MagicMock() - item_1.project_id = "container_id_1" - item_2 = MagicMock() - item_2.project_id = "container_id" - item_3 = MagicMock() - item_3.project_id = "container_id_2" - - item_endpoint.get.side_effect = [ - ([item_1], pagination_item_1), - ([item_2], pagination_item_2), - ([item_3], pagination_item_3), - ] - - result = Server.get_items_by_name(logger, item_endpoint, item_name, container) - - self.assertEqual(result, [item_2]) - self.assertEqual(item_endpoint.get.call_count, 3) - logger.debug.assert_called() - - @patch("tabcmd.commands.server.TSC.RequestOptions") - @patch("tabcmd.commands.server.TSC.Filter") - def test_get_items_by_name_multiple_pages_no_container_match(self, MockFilter, MockRequestOptions): - logger = MagicMock() - item_endpoint = MagicMock() - item_name = "test_item" - container = MagicMock() - container.id = "container_id" - - pagination_item_1 = MagicMock() - pagination_item_1.total_available = 3 - pagination_item_1.page_number = 1 - pagination_item_1.page_size = 1 - - pagination_item_2 = MagicMock() - pagination_item_2.total_available = 3 - pagination_item_2.page_number = 2 - pagination_item_2.page_size = 1 - - pagination_item_3 = MagicMock() - pagination_item_3.total_available = 3 - pagination_item_3.page_number = 3 - pagination_item_3.page_size = 1 - - item_1 = MagicMock() - item_1.project_id = "different_container_id_1" - item_2 = MagicMock() - item_2.project_id = "different_container_id_2" - item_3 = MagicMock() - item_3.project_id = "different_container_id_3" - - item_endpoint.get.side_effect = [ - ([item_1], pagination_item_1), - ([item_2], pagination_item_2), - ([item_3], pagination_item_3), - ] - - result = Server.get_items_by_name(logger, item_endpoint, item_name, container) - - self.assertEqual(result, []) - self.assertEqual(item_endpoint.get.call_count, 3) - logger.debug.assert_called() From ce4048ee0bb3bbf596e6fd76fb72bfd5410b6e20 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Sat, 21 Dec 2024 01:38:38 -0800 Subject: [PATCH 03/17] fix query filter for projects --- tabcmd/commands/server.py | 26 +++++++++++++++----------- tests/commands/test_server.py | 1 - 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/tabcmd/commands/server.py b/tabcmd/commands/server.py index 018e9737..19136141 100644 --- a/tabcmd/commands/server.py +++ b/tabcmd/commands/server.py @@ -61,12 +61,21 @@ def get_items_by_name(logger, item_endpoint, item_name: str, container: Optional req_option.filter.add( TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name) ) + + # todo - this doesn't filter if the project is in the top level. + # todo: there is no guarantee that these fields are the same for different content types. + # probably better if we move that type specific logic out to a wrapper if container: - req_option.filter.add( - TSC.Filter( - TSC.RequestOptions.Field.ParentProjectId, TSC.RequestOptions.Operator.Equals, container.id - ) - ) + # the name of the filter field is different if you are finding a project or any other item + if type(item_endpoint).__name__.find("Projects") < 0: + parentField = TSC.RequestOptions.Field.ProjectName + parentValue = container.name + else: + parentField = TSC.RequestOptions.Field.ParentProjectId + parentValue = container.id + logger.debug("filtering for parent with {}".format(parentField)) + + req_option.filter.add(TSC.Filter(parentField, TSC.RequestOptions.Operator.Equals, parentValue)) all_items, pagination_item = item_endpoint.get(req_option) @@ -165,12 +174,7 @@ def _parse_project_path_to_list(project_path: str): def _get_project_by_name_and_parent(logger, server, project_name: str, parent: Optional[TSC.ProjectItem]): # logger.debug("get by name and parent: {0}, {1}".format(project_name, parent)) # get by name to narrow down the list - projects = Server.get_items_by_name(logger, server.projects, project_name) - if parent is not None: - parent_id = parent.id - for project in projects: - if project.parent_id == parent_id: - return project + projects = Server.get_items_by_name(logger, server.projects, project_name, parent) return projects[0] @staticmethod diff --git a/tests/commands/test_server.py b/tests/commands/test_server.py index 6f8ed058..aaf287e5 100644 --- a/tests/commands/test_server.py +++ b/tests/commands/test_server.py @@ -73,7 +73,6 @@ def test_get_items_by_name_with_container(self, MockFilter, MockRequestOptions): logger.debug.assert_called() item_endpoint.get.assert_called() - @patch("tabcmd.commands.server.TSC.RequestOptions") @patch("tabcmd.commands.server.TSC.Filter") def test_get_items_by_name_multiple_pages(self, MockFilter, MockRequestOptions): From 98af50899f4c9fcd90124d299ce9d616fbea9b79 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Mon, 2 Dec 2024 13:44:02 -0800 Subject: [PATCH 04/17] incremental and sync are not exclusive --- .../commands/extracts/refresh_extracts_command.py | 7 ++----- tabcmd/execution/global_options.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tabcmd/commands/extracts/refresh_extracts_command.py b/tabcmd/commands/extracts/refresh_extracts_command.py index b88f220b..0e4392bb 100644 --- a/tabcmd/commands/extracts/refresh_extracts_command.py +++ b/tabcmd/commands/extracts/refresh_extracts_command.py @@ -22,6 +22,7 @@ def define_args(refresh_extract_parser): set_calculations_options(group) set_project_arg(group) set_parent_project_arg(group) + set_sync_wait_options(group) @staticmethod def run_command(args): @@ -31,14 +32,11 @@ def run_command(args): server = session.create_session(args, logger) if args.addcalculations or args.removecalculations: - logger.warning("Add/Remove Calculations tasks are not yet implemented.") + logger.warning("Add/Remove Calculations tasks are not supported.") - # are these two mandatory? mutually exclusive? # docs: the REST method always runs a full refresh even if the refresh type is set to incremental. if args.incremental: # docs: run the incremental refresh logger.warn("Incremental refresh is not yet available through the new tabcmd") - # if args.synchronous: # docs: run a full refresh and poll until it completes - # else: run a full refresh but don't poll for completion try: item = Extracts.get_wb_or_ds_for_extracts(args, logger, server) @@ -54,7 +52,6 @@ def run_command(args): logger.info(_("common.output.job_queued_success")) logger.debug("Extract refresh queued with JobID: {}".format(job.id)) - if args.synchronous: # maintains a live connection to the server while the refresh operation is underway, polling every second # until the background job is done. diff --git a/tabcmd/execution/global_options.py b/tabcmd/execution/global_options.py index 6c972616..849316ef 100644 --- a/tabcmd/execution/global_options.py +++ b/tabcmd/execution/global_options.py @@ -346,7 +346,7 @@ def set_append_replace_option(parser): ) # what's the difference between this and 'overwrite'? - # This is meant for when a) the local file is an extract b) the server item is an existing data source + # This one replaces the data but not the metadata append_group.add_argument( "--replace", action="store_true", @@ -355,7 +355,7 @@ def set_append_replace_option(parser): ) -# this is meant to be like replacing like +# this is meant to be publish the whole thing on top of what's there def set_overwrite_option(parser): parser.add_argument( "-o", @@ -368,13 +368,16 @@ def set_overwrite_option(parser): # refresh-extracts def set_incremental_options(parser): - sync_group = parser.add_mutually_exclusive_group() - sync_group.add_argument("--incremental", action="store_true", help="Runs the incremental refresh operation.") - sync_group.add_argument( + parser.add_argument("--incremental", action="store_true", help="Runs the incremental refresh operation.") + return parser + + +def set_sync_wait_options(parser): + parser.add_argument( "--synchronous", action="store_true", help="Adds the full refresh operation to the queue used by the Backgrounder process, to be run as soon as a \ - Backgrounder process is available.", + Backgrounder process is available. The program will wait until the job has finished or the timeout has been reached.", ) return parser From 85a75ab7d51ddfc7ef790348bd7bd1aa2a299546 Mon Sep 17 00:00:00 2001 From: Jac Date: Thu, 2 Jan 2025 19:18:15 -0800 Subject: [PATCH 05/17] Fix ubuntu upload --- .github/workflows/package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index d9be64c3..85419485 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -1,4 +1,4 @@ -name: Release-Executable +name: Package-and-Upload # Pyinstaller requires that executables for each OS are built on that OS # This action is intended to build on each of the supported OS's: mac, windows, linux. @@ -107,7 +107,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ./dist/${{ matrix.TARGET }}/${{ matrix.OUT_FILE_NAME }}/ + file: ./dist/${{ matrix.TARGET }}/${{ matrix.OUT_FILE_NAME }} tag: ${{ github.ref_name }} overwrite: true promote: true From e32c4709216f10ccd97f999f978cab4a0c8fe8f5 Mon Sep 17 00:00:00 2001 From: Jac Date: Mon, 6 Jan 2025 15:55:03 -0800 Subject: [PATCH 06/17] jac/auto-package-and-release Edit package workflow to upload executables to release tested in https://github.com/jacalata/tabcmd see workflows at https://github.com/jacalata/tabcmd/actions/runs/12642094818/workflow and uploaded executables at https://github.com/jacalata/tabcmd/releases/tag/development --- .github/workflows/package.yml | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 85419485..43e59ec5 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -1,8 +1,8 @@ name: Package-and-Upload # Pyinstaller requires that executables for each OS are built on that OS -# This action is intended to build on each of the supported OS's: mac, windows, linux. -# and then upload all three files to a new release +# This action is intended to build on each of the supported OS's: mac (x86 and arm), windows, linux. +# and then upload all four files to a new release # reference material: # https://data-dive.com/multi-os-deployment-in-cloud-using-pyinstaller-and-github-actions @@ -27,21 +27,25 @@ jobs: TARGET: windows CMD_BUILD: > pyinstaller tabcmd-windows.spec --clean --noconfirm --distpath ./dist/windows + UPLOAD_FILE_NAME: tabcmd.exe OUT_FILE_NAME: tabcmd.exe ASSET_MIME: application/vnd.microsoft.portable-executable - os: macos-13 TARGET: macos CMD_BUILD: > pyinstaller tabcmd-mac.spec --clean --noconfirm --distpath ./dist/macos/ - BUNDLE_NAME: tabcmd.app + # the extension .app is not allowed as an upload by github + UPLOAD_FILE_NAME: tabcmd-x86.app.tar + # these two names must match the output defined in tabcmd-mac.spec OUT_FILE_NAME: tabcmd.app APP_BINARY_FILE_NAME: tabcmd ASSET_MIME: application/zip - os: macos-latest + # This must match the value set in tabcmd-mac.spec for the output folder TARGET: macos CMD_BUILD: > pyinstaller tabcmd-mac.spec --clean --noconfirm --distpath ./dist/macos/ - BUNDLE_NAME: tabcmd_arm64.app + UPLOAD_FILE_NAME: tabcmd_arm64.app.tar OUT_FILE_NAME: tabcmd.app APP_BINARY_FILE_NAME: tabcmd ASSET_MIME: application/zip @@ -53,6 +57,7 @@ jobs: pyinstaller --clean -y --distpath ./dist/ubuntu tabcmd-linux.spec && chown -R --reference=. ./dist/ubuntu OUT_FILE_NAME: tabcmd + UPLOAD_FILE_NAME: tabcmd steps: - uses: actions/checkout@v4 @@ -87,27 +92,15 @@ jobs: run: | rm -f dist/${{ matrix.TARGET }}/${{ matrix.APP_BINARY_FILE_NAME }} cd dist/${{ matrix.TARGET }} - tar -cvf ${{ matrix.BUNDLE_NAME }}.tar ${{ matrix.OUT_FILE_NAME }} + tar -cvf ${{ matrix.UPLOAD_FILE_NAME }} ${{ matrix.OUT_FILE_NAME }} - - name: Upload artifact - if: matrix.TARGET != 'macos' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.OUT_FILE_NAME }} - path: ./dist/${{ matrix.TARGET }}/${{ matrix.OUT_FILE_NAME }} - - - name: Upload artifact for Mac - if: matrix.TARGET == 'macos' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.BUNDLE_NAME }} - path: ./dist/${{ matrix.TARGET }}/${{ matrix.BUNDLE_NAME }}.tar - - name: Upload binaries to release + - name: Upload binaries to release for ${{ matrix.TARGET }} uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ./dist/${{ matrix.TARGET }}/${{ matrix.OUT_FILE_NAME }} + asset_name: ${{ matrix.UPLOAD_FILE_NAME }} + file: ./dist/${{ matrix.TARGET }}/${{ matrix.UPLOAD_FILE_NAME }} tag: ${{ github.ref_name }} overwrite: true promote: true From 0f0e093365043b26d01dd0699f2a575ea6e82550 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Mon, 6 Jan 2025 23:18:19 -0800 Subject: [PATCH 07/17] Update tsc to 0.35 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a8839e58..b1fe6e0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ dependencies = [ "types-mock", "types-requests", "types-setuptools", - "tableauserverclient==0.34", + "tableauserverclient==0.35", "urllib3", ] [project.optional-dependencies] From 1908a38c429bedde99f9b0f26fa1e61f2afd6a3c Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Mon, 6 Jan 2025 23:33:42 -0800 Subject: [PATCH 08/17] update publish workbook --- .../publish_command.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tabcmd/commands/datasources_and_workbooks/publish_command.py b/tabcmd/commands/datasources_and_workbooks/publish_command.py index a4ff8c26..cdf1ec8d 100644 --- a/tabcmd/commands/datasources_and_workbooks/publish_command.py +++ b/tabcmd/commands/datasources_and_workbooks/publish_command.py @@ -54,14 +54,23 @@ def run_command(args): publish_mode = PublishCommand.get_publish_mode(args, logger) + connection = TSC.models.ConnectionItem() if args.db_username: - creds = TSC.models.ConnectionCredentials(args.db_username, args.db_password, embed=args.save_db_password) + connection.connection_credentials = TSC.models.ConnectionCredentials(args.db_username, args.db_password, embed=args.save_db_password) + connections.append(connection) elif args.oauth_username: - creds = TSC.models.ConnectionCredentials(args.oauth_username, None, embed=False, oauth=args.save_oauth) + connection.connection_credentials = TSC.models.ConnectionCredentials(args.oauth_username, None, embed=False, oauth=args.save_oauth) + connections.append(connection) else: logger.debug("No db-username or oauth-username found in command") - creds = None + connection = None + if connection: + connections = list() + connections.append(connection) + else: + connections = None + source = PublishCommand.get_filename_extension_if_tableau_type(logger, args.filename) logger.info(_("publish.status").format(args.filename)) if source in ["twbx", "twb"]: @@ -76,9 +85,7 @@ def run_command(args): new_workbook, args.filename, publish_mode, - # args.thumbnail_username, not yet implemented in tsc - # args.thumbnail_group, - connection_credentials=creds, + connections=connections, as_job=False, skip_connection_check=args.skip_connection_check, ) @@ -92,7 +99,7 @@ def run_command(args): new_datasource.use_remote_query_agent = args.use_tableau_bridge try: new_datasource = server.datasources.publish( - new_datasource, args.filename, publish_mode, connection_credentials=creds + new_datasource, args.filename, publish_mode, connections=connections ) except Exception as exc: Errors.exit_with_error(logger, exception=exc) From 1246ab4bdb964232d94e760f97f689607f1544be Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Mon, 6 Jan 2025 23:18:19 -0800 Subject: [PATCH 09/17] Update tsc to 0.35 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a8839e58..b1fe6e0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ dependencies = [ "types-mock", "types-requests", "types-setuptools", - "tableauserverclient==0.34", + "tableauserverclient==0.35", "urllib3", ] [project.optional-dependencies] From 8aef27fb950e36f876a27f640381eb7ea5beb9ef Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Mon, 6 Jan 2025 23:46:03 -0800 Subject: [PATCH 10/17] Update publish_command.py --- tabcmd/commands/datasources_and_workbooks/publish_command.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tabcmd/commands/datasources_and_workbooks/publish_command.py b/tabcmd/commands/datasources_and_workbooks/publish_command.py index cdf1ec8d..ab940e19 100644 --- a/tabcmd/commands/datasources_and_workbooks/publish_command.py +++ b/tabcmd/commands/datasources_and_workbooks/publish_command.py @@ -57,10 +57,8 @@ def run_command(args): connection = TSC.models.ConnectionItem() if args.db_username: connection.connection_credentials = TSC.models.ConnectionCredentials(args.db_username, args.db_password, embed=args.save_db_password) - connections.append(connection) elif args.oauth_username: connection.connection_credentials = TSC.models.ConnectionCredentials(args.oauth_username, None, embed=False, oauth=args.save_oauth) - connections.append(connection) else: logger.debug("No db-username or oauth-username found in command") connection = None From a65cb8d38dba334df2551450a9d2887d2bb5e841 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Mon, 6 Jan 2025 23:46:28 -0800 Subject: [PATCH 11/17] format --- .../datasources_and_workbooks/publish_command.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tabcmd/commands/datasources_and_workbooks/publish_command.py b/tabcmd/commands/datasources_and_workbooks/publish_command.py index ab940e19..98a3e5e9 100644 --- a/tabcmd/commands/datasources_and_workbooks/publish_command.py +++ b/tabcmd/commands/datasources_and_workbooks/publish_command.py @@ -56,9 +56,13 @@ def run_command(args): connection = TSC.models.ConnectionItem() if args.db_username: - connection.connection_credentials = TSC.models.ConnectionCredentials(args.db_username, args.db_password, embed=args.save_db_password) + connection.connection_credentials = TSC.models.ConnectionCredentials( + args.db_username, args.db_password, embed=args.save_db_password + ) elif args.oauth_username: - connection.connection_credentials = TSC.models.ConnectionCredentials(args.oauth_username, None, embed=False, oauth=args.save_oauth) + connection.connection_credentials = TSC.models.ConnectionCredentials( + args.oauth_username, None, embed=False, oauth=args.save_oauth + ) else: logger.debug("No db-username or oauth-username found in command") connection = None @@ -68,7 +72,7 @@ def run_command(args): connections.append(connection) else: connections = None - + source = PublishCommand.get_filename_extension_if_tableau_type(logger, args.filename) logger.info(_("publish.status").format(args.filename)) if source in ["twbx", "twb"]: From 82c375a9bdddeddcf7c35d36a6d69e764890e894 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Thu, 13 Jun 2024 23:44:35 -0700 Subject: [PATCH 12/17] add space to --version output --- tabcmd/execution/parent_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tabcmd/execution/parent_parser.py b/tabcmd/execution/parent_parser.py index 41362bed..df9b3888 100644 --- a/tabcmd/execution/parent_parser.py +++ b/tabcmd/execution/parent_parser.py @@ -131,7 +131,7 @@ def parent_parser_with_global_options(): "-v", "--version", action="version", - version=strings[6] + "v" + version + "\n \n", + version=strings[6] + " v" + version + "\n \n", help=strings[7], ) From e7f4993ccded39581973cf1b6cc82acb52b67062 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Thu, 13 Jun 2024 23:37:19 -0700 Subject: [PATCH 13/17] if no command is given, print help info --- tabcmd/execution/parent_parser.py | 1 + tabcmd/execution/tabcmd_controller.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tabcmd/execution/parent_parser.py b/tabcmd/execution/parent_parser.py index df9b3888..15216631 100644 --- a/tabcmd/execution/parent_parser.py +++ b/tabcmd/execution/parent_parser.py @@ -182,6 +182,7 @@ def include(self, command): command.define_args(additional_parser) return additional_parser + # Help isn't added like the others because it has to have access to the rest, to get their args def include_help(self): additional_parser = self.subparsers.add_parser("help", help=strings[14], parents=[self.global_options]) additional_parser._optionals.title = strings[1] diff --git a/tabcmd/execution/tabcmd_controller.py b/tabcmd/execution/tabcmd_controller.py index 2fbf270d..5487e536 100644 --- a/tabcmd/execution/tabcmd_controller.py +++ b/tabcmd/execution/tabcmd_controller.py @@ -26,32 +26,32 @@ def run(parser, user_input=None): sys.exit(0) user_input = user_input or sys.argv[1:] namespace = parser.parse_args(user_input) + # if no subcommand was given, call help + if not hasattr(namespace, "func"): + print("No command found.") + parser.print_help() + sys.exit(0) + if hasattr("namespace", "logging_level") and namespace.logging_level != logging.INFO: print("logging:", namespace.logging_level) logger = log(__name__, namespace.logging_level or logging.INFO) logger.info("Tabcmd {}".format(version)) - if (hasattr("namespace", "password") or hasattr("namespace", "token_value")) and hasattr("namespace", "func"): + if hasattr(namespace, "password") or hasattr(namespace, "token_value"): # don't print whole namespace because it has secrets logger.debug(namespace.func) else: logger.debug(namespace) - if namespace.language: + if hasattr(namespace, "language"): set_client_locale(namespace.language, logger) if namespace.query_page_size: os.environ["TSC_PAGE_SIZE"] = str(namespace.query_page_size) try: - func = namespace.func - # if a subcommand was identified, call the function assigned to it - # this is the functional equivalent of the call by reflection in the previous structure # https://stackoverflow.com/questions/49038616/argparse-subparsers-with-functions namespace.func.run_command(namespace) - except AttributeError: - parser.error("No command identified or too few arguments") except Exception as e: # todo: use log_stack here for better presentation? logger.exception(e) - # if no command was given, argparse will just not create the attribute sys.exit(2) return namespace From f2a7b6dfdd1e324d07bbf06e94a8759dbab91277 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Thu, 13 Jun 2024 23:37:19 -0700 Subject: [PATCH 14/17] if no command is given, print help info --- tabcmd/execution/tabcmd_controller.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tabcmd/execution/tabcmd_controller.py b/tabcmd/execution/tabcmd_controller.py index 5487e536..684d0495 100644 --- a/tabcmd/execution/tabcmd_controller.py +++ b/tabcmd/execution/tabcmd_controller.py @@ -46,6 +46,7 @@ def run(parser, user_input=None): set_client_locale(namespace.language, logger) if namespace.query_page_size: os.environ["TSC_PAGE_SIZE"] = str(namespace.query_page_size) + try: # https://stackoverflow.com/questions/49038616/argparse-subparsers-with-functions namespace.func.run_command(namespace) From 2d2238e3ce27fc36782e740ea45ee2c014ef1a70 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Fri, 24 Mar 2023 13:55:15 -0700 Subject: [PATCH 15/17] update version number in exe --- res/metadata.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/res/metadata.yml b/res/metadata.yml index e02c9533..2d34a5a0 100644 --- a/res/metadata.yml +++ b/res/metadata.yml @@ -1,4 +1,4 @@ -Version: 0.0.0.98765 +Version: 2.0.0.15 CompanyName: Salesforce, Inc. FileDescription: Tabcmd - CLI for Tableau InternalName: Tabcmd From ea50e62d94e28d28cce0c5a3ed05c9c0b8891b11 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Wed, 8 Jan 2025 13:20:39 -0800 Subject: [PATCH 16/17] bump version in windows exe Before our next release, this should be edited so it reads the version from the git tags like everywhere else. --- res/metadata.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/res/metadata.yml b/res/metadata.yml index 2d34a5a0..5223288f 100644 --- a/res/metadata.yml +++ b/res/metadata.yml @@ -1,4 +1,4 @@ -Version: 2.0.0.15 +Version: 2.0.0.16 CompanyName: Salesforce, Inc. FileDescription: Tabcmd - CLI for Tableau InternalName: Tabcmd From 0c107c9a627e8202913e394122958a95e11f43ba Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Wed, 8 Jan 2025 13:22:24 -0800 Subject: [PATCH 17/17] fix from andy's comment --- tabcmd/execution/tabcmd_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tabcmd/execution/tabcmd_controller.py b/tabcmd/execution/tabcmd_controller.py index 684d0495..3b544d2c 100644 --- a/tabcmd/execution/tabcmd_controller.py +++ b/tabcmd/execution/tabcmd_controller.py @@ -32,7 +32,7 @@ def run(parser, user_input=None): parser.print_help() sys.exit(0) - if hasattr("namespace", "logging_level") and namespace.logging_level != logging.INFO: + if hasattr(namespace, "logging_level") and namespace.logging_level != logging.INFO: print("logging:", namespace.logging_level) logger = log(__name__, namespace.logging_level or logging.INFO)