Promote code for 2.0.16 (#335)

- filter workbooks for export
- incremental extract refresh
- publish with replacement

.github/workflows/package.yml

@@ -1,8 +1,8 @@
-name: Release-Executable
+name: Package-and-Upload
 
 # Pyinstaller requires that executables for each OS are built on that OS
-# This action is intended to build on each of the supported OS's: mac, windows, linux.
-# and then upload all three files to a new release
+# This action is intended to build on each of the supported OS's: mac (x86 and arm), windows, linux.
+# and then upload all four files to a new release
 
 # reference material:
 # https://data-dive.com/multi-os-deployment-in-cloud-using-pyinstaller-and-github-actions
@@ -27,21 +27,25 @@ jobs:
             TARGET: windows
             CMD_BUILD: >
               pyinstaller tabcmd-windows.spec --clean --noconfirm --distpath ./dist/windows
+            UPLOAD_FILE_NAME: tabcmd.exe
             OUT_FILE_NAME: tabcmd.exe
             ASSET_MIME: application/vnd.microsoft.portable-executable
           - os: macos-13
             TARGET: macos
             CMD_BUILD:  >
               pyinstaller tabcmd-mac.spec --clean --noconfirm --distpath ./dist/macos/
-            BUNDLE_NAME: tabcmd.app
+            # the extension .app is not allowed as an upload by github
+            UPLOAD_FILE_NAME: tabcmd-x86.app.tar
+            # these two names must match the output defined in tabcmd-mac.spec
             OUT_FILE_NAME: tabcmd.app
             APP_BINARY_FILE_NAME: tabcmd
             ASSET_MIME: application/zip
           - os: macos-latest
+            # This must match the value set in tabcmd-mac.spec for the output folder
             TARGET: macos
             CMD_BUILD:  >
               pyinstaller tabcmd-mac.spec --clean --noconfirm --distpath ./dist/macos/
-            BUNDLE_NAME: tabcmd_arm64.app
+            UPLOAD_FILE_NAME: tabcmd_arm64.app.tar
             OUT_FILE_NAME: tabcmd.app
             APP_BINARY_FILE_NAME: tabcmd
             ASSET_MIME: application/zip
@@ -53,6 +57,7 @@ jobs:
               pyinstaller --clean -y --distpath ./dist/ubuntu tabcmd-linux.spec &&
               chown -R --reference=. ./dist/ubuntu
             OUT_FILE_NAME: tabcmd
+            UPLOAD_FILE_NAME: tabcmd
 
     steps:
     - uses: actions/checkout@v4
@@ -87,27 +92,15 @@ jobs:
       run: |
         rm -f dist/${{ matrix.TARGET }}/${{ matrix.APP_BINARY_FILE_NAME }}
         cd dist/${{ matrix.TARGET }}
-        tar -cvf ${{ matrix.BUNDLE_NAME }}.tar ${{ matrix.OUT_FILE_NAME }}
+        tar -cvf ${{ matrix.UPLOAD_FILE_NAME }} ${{ matrix.OUT_FILE_NAME }}
 
-    - name: Upload artifact
-      if: matrix.TARGET != 'macos'
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ matrix.OUT_FILE_NAME }}
-        path: ./dist/${{ matrix.TARGET }}/${{ matrix.OUT_FILE_NAME }}
-
-    - name: Upload artifact for Mac 
-      if: matrix.TARGET == 'macos'
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ matrix.BUNDLE_NAME }}
-        path: ./dist/${{ matrix.TARGET }}/${{ matrix.BUNDLE_NAME }}.tar
     
-    - name: Upload binaries to release
+    - name: Upload binaries to release for ${{ matrix.TARGET }}
       uses: svenstaro/upload-release-action@v2
       with:
         repo_token: ${{ secrets.GITHUB_TOKEN }}
-        file: ./dist/${{ matrix.TARGET }}/${{ matrix.OUT_FILE_NAME }}/
+        asset_name: ${{ matrix.UPLOAD_FILE_NAME }}
+        file: ./dist/${{ matrix.TARGET }}/${{ matrix.UPLOAD_FILE_NAME }}
         tag: ${{ github.ref_name }}
         overwrite: true
         promote: true

pyproject.toml

@@ -52,7 +52,7 @@ dependencies = [
     "types-mock",
     "types-requests",
     "types-setuptools",
-    "tableauserverclient==0.34",
+    "tableauserverclient==0.35",
     "urllib3",
 ]
 [project.optional-dependencies]

res/metadata.yml

@@ -1,4 +1,4 @@
-Version: 0.0.0.98765
+Version: 2.0.0.16
 CompanyName: Salesforce, Inc.
 FileDescription: Tabcmd - CLI for Tableau
 InternalName: Tabcmd

tabcmd/commands/datasources_and_workbooks/publish_command.py

@@ -54,13 +54,24 @@ def run_command(args):
 
         publish_mode = PublishCommand.get_publish_mode(args, logger)
 
+        connection = TSC.models.ConnectionItem()
         if args.db_username:
-            creds = TSC.models.ConnectionCredentials(args.db_username, args.db_password, embed=args.save_db_password)
+            connection.connection_credentials = TSC.models.ConnectionCredentials(
+                args.db_username, args.db_password, embed=args.save_db_password
+            )
         elif args.oauth_username:
-            creds = TSC.models.ConnectionCredentials(args.oauth_username, None, embed=False, oauth=args.save_oauth)
+            connection.connection_credentials = TSC.models.ConnectionCredentials(
+                args.oauth_username, None, embed=False, oauth=args.save_oauth
+            )
         else:
             logger.debug("No db-username or oauth-username found in command")
-            creds = None
+            connection = None
+
+        if connection:
+            connections = list()
+            connections.append(connection)
+        else:
+            connections = None
 
         source = PublishCommand.get_filename_extension_if_tableau_type(logger, args.filename)
         logger.info(_("publish.status").format(args.filename))
@@ -76,9 +87,7 @@ def run_command(args):
                     new_workbook,
                     args.filename,
                     publish_mode,
-                    # args.thumbnail_username, not yet implemented in tsc
-                    # args.thumbnail_group,
-                    connection_credentials=creds,
+                    connections=connections,
                     as_job=False,
                     skip_connection_check=args.skip_connection_check,
                 )
@@ -92,7 +101,7 @@ def run_command(args):
             new_datasource.use_remote_query_agent = args.use_tableau_bridge
             try:
                 new_datasource = server.datasources.publish(
-                    new_datasource, args.filename, publish_mode, connection_credentials=creds
+                    new_datasource, args.filename, publish_mode, connections=connections
                 )
             except Exception as exc:
                 Errors.exit_with_error(logger, exception=exc)

tabcmd/commands/extracts/refresh_extracts_command.py

@@ -22,6 +22,7 @@ def define_args(refresh_extract_parser):
         set_calculations_options(group)
         set_project_arg(group)
         set_parent_project_arg(group)
+        set_sync_wait_options(group)
 
     @staticmethod
     def run_command(args):
@@ -31,14 +32,11 @@ def run_command(args):
         server = session.create_session(args, logger)
 
         if args.addcalculations or args.removecalculations:
-            logger.warning("Add/Remove Calculations tasks are not yet implemented.")
+            logger.warning("Add/Remove Calculations tasks are not supported.")
 
-        # are these two mandatory? mutually exclusive?
         # docs: the REST method always runs a full refresh even if the refresh type is set to incremental.
         if args.incremental:  # docs: run the incremental refresh
             logger.warn("Incremental refresh is not yet available through the new tabcmd")
-        # if args.synchronous:  # docs: run a full refresh and poll until it completes
-        # else:  run a full refresh but don't poll for completion
 
         try:
             item = Extracts.get_wb_or_ds_for_extracts(args, logger, server)
@@ -54,7 +52,6 @@ def run_command(args):
 
         logger.info(_("common.output.job_queued_success"))
         logger.debug("Extract refresh queued with JobID: {}".format(job.id))
-
         if args.synchronous:
             # maintains a live connection to the server while the refresh operation is underway, polling every second
             # until the background job is done.   <job id="JOB_ID" mode="MODE" type="RefreshExtract" />

tabcmd/commands/server.py

@@ -50,31 +50,59 @@ def get_items_by_name(logger, item_endpoint, item_name: str, container: Optional
             container_name: str = "[{0}] {1}".format(container.__class__.__name__, container.name)
             item_log_name = "{0}/{1}".format(container_name, item_log_name)
         logger.debug(_("export.status").format(item_log_name))
-        req_option = TSC.RequestOptions()
-        req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name))
-        all_items, pagination_item = item_endpoint.get(req_option)
-        if all_items is None or all_items == []:
-            raise TSC.ServerResponseError(
-                code="404",
-                summary=_("errors.xmlapi.not_found"),
-                detail=_("errors.xmlapi.not_found") + ": " + item_log_name,
+
+        result = []
+        total_available_items = None
+        page_number = 1
+        total_retrieved_items = 0
+
+        while True:
+            req_option = TSC.RequestOptions(pagenumber=page_number)
+            req_option.filter.add(
+                TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name)
             )
-        if len(all_items) == 1:
-            logger.debug("Exactly one result found")
-            result = all_items
-        if len(all_items) > 1:
+
+            # todo - this doesn't filter if the project is in the top level.
+            # todo: there is no guarantee that these fields are the same for different content types.
+            # probably better if we move that type specific logic out to a wrapper
+            if container:
+                # the name of the filter field is different if you are finding a project or any other item
+                if type(item_endpoint).__name__.find("Projects") < 0:
+                    parentField = TSC.RequestOptions.Field.ProjectName
+                    parentValue = container.name
+                else:
+                    parentField = TSC.RequestOptions.Field.ParentProjectId
+                    parentValue = container.id
+                logger.debug("filtering for parent with {}".format(parentField))
+
+                req_option.filter.add(TSC.Filter(parentField, TSC.RequestOptions.Operator.Equals, parentValue))
+
+            all_items, pagination_item = item_endpoint.get(req_option)
+
+            if pagination_item.total_available == 0:
+                raise TSC.ServerResponseError(
+                    code="404",
+                    summary=_("errors.xmlapi.not_found"),
+                    detail=_("errors.xmlapi.not_found") + ": " + item_log_name,
+                )
+
+            total_retrieved_items += len(all_items)
+
             logger.debug(
-                "{}+ items of this name were found: {}".format(
-                    len(all_items), all_items[0].name + ", " + all_items[1].name + ", ..."
+                "{} items of name: {} were found for query page number: {}, page size: {} & total available: {}".format(
+                    len(all_items),
+                    item_name,
+                    pagination_item.page_number,
+                    pagination_item.page_size,
+                    pagination_item.total_available,
                 )
             )
 
-            if container:
-                container_id = container.id
-                logger.debug("Filtering to items in project {}".format(container.id))
-                result = list(filter(lambda item: item.project_id == container_id, all_items))
-            else:
-                result = all_items
+            result.extend(all_items)
+            if total_retrieved_items >= pagination_item.total_available:
+                break
+
+            page_number = pagination_item.page_number + 1
 
         return result
 
@@ -146,12 +174,7 @@ def _parse_project_path_to_list(project_path: str):
     def _get_project_by_name_and_parent(logger, server, project_name: str, parent: Optional[TSC.ProjectItem]):
         # logger.debug("get by name and parent: {0}, {1}".format(project_name, parent))
         # get by name to narrow down the list
-        projects = Server.get_items_by_name(logger, server.projects, project_name)
-        if parent is not None:
-            parent_id = parent.id
-            for project in projects:
-                if project.parent_id == parent_id:
-                    return project
+        projects = Server.get_items_by_name(logger, server.projects, project_name, parent)
         return projects[0]
 
     @staticmethod

tabcmd/execution/global_options.py

@@ -346,7 +346,7 @@ def set_append_replace_option(parser):
     )
 
     # what's the difference between this and 'overwrite'?
-    # This is meant for when a) the local file is an extract b) the server item is an existing data source
+    # This one replaces the data but not the metadata
     append_group.add_argument(
         "--replace",
         action="store_true",
@@ -355,7 +355,7 @@ def set_append_replace_option(parser):
     )
 
 
-# this is meant to be like replacing like
+# this is meant to be publish the whole thing on top of what's there
 def set_overwrite_option(parser):
     parser.add_argument(
         "-o",
@@ -368,13 +368,16 @@ def set_overwrite_option(parser):
 
 # refresh-extracts
 def set_incremental_options(parser):
-    sync_group = parser.add_mutually_exclusive_group()
-    sync_group.add_argument("--incremental", action="store_true", help="Runs the incremental refresh operation.")
-    sync_group.add_argument(
+    parser.add_argument("--incremental", action="store_true", help="Runs the incremental refresh operation.")
+    return parser
+
+
+def set_sync_wait_options(parser):
+    parser.add_argument(
         "--synchronous",
         action="store_true",
         help="Adds the full refresh operation to the queue used by the Backgrounder process, to be run as soon as a \
-        Backgrounder process is available.",
+        Backgrounder process is available. The program will wait until the job has finished or the timeout has been reached.",
     )
     return parser
 

tabcmd/execution/parent_parser.py

@@ -131,9 +131,17 @@ def parent_parser_with_global_options():
         "-v",
         "--version",
         action="version",
-        version=strings[6] + "v" + version + "\n \n",
+        version=strings[6] + " v" + version + "\n \n",
         help=strings[7],
     )
+
+    parser.add_argument(
+        "--query-page-size",
+        type=int,
+        default=None,
+        metavar="<PAGE_SIZE>",
+        help="Specify the page size for query results.",
+    )
     return parser
 
 
@@ -174,6 +182,7 @@ def include(self, command):
         command.define_args(additional_parser)
         return additional_parser
 
+    # Help isn't added like the others because it has to have access to the rest, to get their args
     def include_help(self):
         additional_parser = self.subparsers.add_parser("help", help=strings[14], parents=[self.global_options])
         additional_parser._optionals.title = strings[1]

tabcmd/execution/tabcmd_controller.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import sys
 
 from .localize import set_client_locale
@@ -25,30 +26,33 @@ def run(parser, user_input=None):
             sys.exit(0)
         user_input = user_input or sys.argv[1:]
         namespace = parser.parse_args(user_input)
-        if hasattr("namespace", "logging_level") and namespace.logging_level != logging.INFO:
+        # if no subcommand was given, call help
+        if not hasattr(namespace, "func"):
+            print("No command found.")
+            parser.print_help()
+            sys.exit(0)
+
+        if hasattr(namespace, "logging_level") and namespace.logging_level != logging.INFO:
             print("logging:", namespace.logging_level)
 
         logger = log(__name__, namespace.logging_level or logging.INFO)
         logger.info("Tabcmd {}".format(version))
-        if (hasattr("namespace", "password") or hasattr("namespace", "token_value")) and hasattr("namespace", "func"):
+        if hasattr(namespace, "password") or hasattr(namespace, "token_value"):
             # don't print whole namespace because it has secrets
             logger.debug(namespace.func)
         else:
             logger.debug(namespace)
-        if namespace.language:
+        if hasattr(namespace, "language"):
             set_client_locale(namespace.language, logger)
+        if namespace.query_page_size:
+            os.environ["TSC_PAGE_SIZE"] = str(namespace.query_page_size)
+
         try:
-            func = namespace.func
-            # if a subcommand was identified, call the function assigned to it
-            # this is the functional equivalent of the call by reflection in the previous structure
             # https://stackoverflow.com/questions/49038616/argparse-subparsers-with-functions
             namespace.func.run_command(namespace)
-        except AttributeError:
-            parser.error("No command identified or too few arguments")
         except Exception as e:
             # todo: use log_stack here for better presentation?
             logger.exception(e)
-            # if no command was given, argparse will just not create the attribute
             sys.exit(2)
 
         return namespace

tests/commands/test_projects_utils.py

@@ -7,7 +7,11 @@
 fake_item = mock.MagicMock()
 fake_item.name = "fake-name"
 fake_item.id = "fake-id"
-getter = mock.MagicMock("get", return_value=([fake_item], 1))
+fake_item_pagination = mock.MagicMock()
+fake_item_pagination.page_number = 1
+fake_item_pagination.total_available = 1
+fake_item_pagination.page_size = 100
+getter = mock.MagicMock("get", return_value=([fake_item], fake_item_pagination))
 
 
 class ProjectsTest(unittest.TestCase):