diff --git a/src/gam.py b/src/gam.py index c339141fe..45ccc14c6 100755 --- a/src/gam.py +++ b/src/gam.py @@ -894,107 +894,6 @@ def getSvcAcctCredentials(scopes, act_as): printLine(MESSAGE_INSTRUCTIONS_OAUTH2SERVICE_JSON) controlflow.invalid_json_exit(GC_Values[GC_OAUTH2SERVICE_JSON]) -def getPageSize(service, function, kwargs): - """Gets maximum maxResults value for API call. Uses value from discovery if - it exists, otherwise value from MAX_RESULTS_API_EXCEPTIONS, otherwise None""" - - method = getattr(service, function) - api_id = method(**kwargs).methodId - for resource in service._rootDesc.get('resources', {}).values(): - for a_method in resource.get('methods', {}).values(): - if a_method.get('id') == api_id: - if not a_method.get('parameters') or a_method['parameters'].get('pageSize') or not a_method['parameters'].get('maxResults'): - # make sure API call supports maxResults. For now we don't care to - # set pageSize since all known pageSize API calls have - # default pageSize == max pageSize - return - return {'maxResults': a_method['parameters']['maxResults'].get('maximum', MAX_RESULTS_API_EXCEPTIONS.get(api_id, None))} - -def callGAPIpages(service, function, items='items', - page_message=None, message_attribute=None, - soft_errors=False, throw_reasons=None, retry_reasons=None, - **kwargs): - """Aggregates and returns all pages of a Google service function response. - - All pages of items are aggregated and returned as a single list. - - Args: - service: A Google service object for the desired API. - function: String, The name of a service request method to execute. - items: String, the name of the resulting "items" field within the method's - response object. The items in this field will be aggregated across all - pages and returned. - page_message: String, a message to be displayed to the user during paging. - Template strings allow for dynamic content to be inserted during paging. - - Supported template strings: - %%total_items%% : The current number of items discovered across all - pages. - %%first_item%% : In conjunction with `message_attribute` arg, will - display a unique property of the first item in the - current page. - %%last_item%% : In conjunction with `message_attribute` arg, will - display a unique property of the last item in the - current page. - - message_attribute: String, the name of a signature field within a single - returned item which identifies that unique item. This field is used with - `page_message` to templatize a paging status message. - soft_errors: Bool, If True, writes non-fatal errors to stderr. - throw_reasons: A list of Google HTTP error reason strings indicating the - errors generated by this request should be re-thrown. All other HTTP - errors are consumed. - retry_reasons: A list of Google HTTP error reason strings indicating which - error should be retried, using exponential backoff techniques, when the - error reason is encountered. - - Returns: - A list of all items received from all paged responses. - """ - if 'maxResults' not in kwargs and 'pageSize' not in kwargs: - page_key = getPageSize(service, function, kwargs) - if page_key: - kwargs.update(page_key) - all_items = [] - page_token = None - total_items = 0 - while True: - page = gapi.call(service, - function, - soft_errors=soft_errors, - throw_reasons=throw_reasons, - retry_reasons=retry_reasons, - pageToken=page_token, - **kwargs) - if page: - page_token = page.get('nextPageToken') - page_items = page.get(items, []) - num_page_items = len(page_items) - total_items += num_page_items - all_items.extend(page_items) - else: - page_token = None - num_page_items = 0 - - # Show a paging message to the user that indicates paging progress - if page_message: - show_message = page_message.replace('%%total_items%%', str(total_items)) - if message_attribute: - first_item = page_items[0] if num_page_items > 0 else {} - last_item = page_items[-1] if num_page_items > 1 else first_item - show_message = show_message.replace('%%first_item%%', str(first_item.get(message_attribute, ''))) - show_message = show_message.replace('%%last_item%%', str(last_item.get(message_attribute, ''))) - sys.stderr.write('\r') - sys.stderr.flush() - sys.stderr.write(show_message) - - if not page_token: - # End the paging status message and return all items. - if page_message and (page_message[-1] != '\n'): - sys.stderr.write('\r\n') - sys.stderr.flush() - return all_items - def getAPIVersion(api): version = API_VER_MAPPING.get(api, 'v1') if api in ['directory', 'reports', 'datatransfer']: @@ -1454,7 +1353,7 @@ def showReport(): while True: try: if fullDataRequired is not None: - warnings = gapi.get_first_page( + warnings = gapi.get_items( rep.userUsageReport(), 'get', 'warnings', throw_reasons=[gapi.errors.ErrorReason.INVALID], date=tryDate, userKey=userKey, customerId=customerId, orgUnitID=orgUnitId, fields='warnings') @@ -1465,8 +1364,8 @@ def showReport(): if fullData == 0: continue page_message = 'Got %%total_items%% Users\n' - usage = callGAPIpages(rep.userUsageReport(), 'get', 'usageReports', page_message=page_message, throw_reasons=[gapi.errors.ErrorReason.INVALID], - date=tryDate, userKey=userKey, customerId=customerId, orgUnitID=orgUnitId, filters=filters, parameters=parameters) + usage = gapi.get_all_pages(rep.userUsageReport(), 'get', 'usageReports', page_message=page_message, throw_reasons=[gapi.errors.ErrorReason.INVALID], + date=tryDate, userKey=userKey, customerId=customerId, orgUnitID=orgUnitId, filters=filters, parameters=parameters) break except gapi.errors.GapiInvalidError as e: tryDate = _adjustDate(str(e)) @@ -1497,7 +1396,7 @@ def showReport(): while True: try: if fullDataRequired is not None: - warnings = gapi.get_first_page( + warnings = gapi.get_items( rep.customerUsageReports(), 'get', 'warnings', throw_reasons=[gapi.errors.ErrorReason.INVALID], customerId=customerId, date=tryDate, fields='warnings') @@ -1507,8 +1406,8 @@ def showReport(): sys.exit(1) if fullData == 0: continue - usage = callGAPIpages(rep.customerUsageReports(), 'get', 'usageReports', throw_reasons=[gapi.errors.ErrorReason.INVALID], - customerId=customerId, date=tryDate, parameters=parameters) + usage = gapi.get_all_pages(rep.customerUsageReports(), 'get', 'usageReports', throw_reasons=[gapi.errors.ErrorReason.INVALID], + customerId=customerId, date=tryDate, parameters=parameters) break except gapi.errors.GapiInvalidError as e: tryDate = _adjustDate(str(e)) @@ -1571,14 +1470,14 @@ def showReport(): elif report == 'group': report = 'groups' page_message = 'Got %%total_items%% items\n' - activities = callGAPIpages(rep.activities(), 'list', 'items', - page_message=page_message, - applicationName=report, userKey=userKey, - customerId=customerId, - actorIpAddress=actorIpAddress, - startTime=startTime, endTime=endTime, - eventName=eventName, filters=filters, - orgUnitID=orgUnitId) + activities = gapi.get_all_pages(rep.activities(), 'list', 'items', + page_message=page_message, + applicationName=report, userKey=userKey, + customerId=customerId, + actorIpAddress=actorIpAddress, + startTime=startTime, endTime=endTime, + eventName=eventName, filters=filters, + orgUnitID=orgUnitId) if activities: titles = ['name'] csvRows = [] @@ -1632,7 +1531,7 @@ def watchGmail(users): gamTopics = project+'/topics/gam-pubsub-gmail-' gamSubscriptions = project+'/subscriptions/gam-pubsub-gmail-' pubsub = buildGAPIObject('pubsub') - topics = callGAPIpages(pubsub.projects().topics(), 'list', items='topics', project=project) + topics = gapi.get_all_pages(pubsub.projects().topics(), 'list', items='topics', project=project) for atopic in topics: if atopic['name'].startswith(gamTopics): topic = atopic['name'] @@ -1642,7 +1541,7 @@ def watchGmail(users): gapi.call(pubsub.projects().topics(), 'create', name=topic) body = {'policy': {'bindings': [{'members': ['serviceAccount:gmail-api-push@system.gserviceaccount.com'], 'role': 'roles/pubsub.editor'}]}} gapi.call(pubsub.projects().topics(), 'setIamPolicy', resource=topic, body=body) - subscriptions = callGAPIpages(pubsub.projects().topics().subscriptions(), 'list', items='subscriptions', topic=topic) + subscriptions = gapi.get_all_pages(pubsub.projects().topics().subscriptions(), 'list', items='subscriptions', topic=topic) for asubscription in subscriptions: if asubscription.startswith(gamSubscriptions): subscription = asubscription @@ -1944,9 +1843,9 @@ def doGetCustomerInfo(): # of current primary being added, not customer create date. # We should also get all domains and use oldest date oldest = datetime.datetime.strptime(customer_info['customerCreationTime'], '%Y-%m-%dT%H:%M:%S.%fZ') - domains = gapi.get_first_page(cd.domains(), 'list', 'domains', - customer=GC_Values[GC_CUSTOMER_ID], - fields='domains(creationTime)') + domains = gapi.get_items(cd.domains(), 'list', 'domains', + customer=GC_Values[GC_CUSTOMER_ID], + fields='domains(creationTime)') for domain in domains: domain_creation = datetime.datetime.fromtimestamp(int(domain['creationTime'])/1000) if domain_creation < oldest: @@ -1979,8 +1878,8 @@ def doGetCustomerInfo(): usage = None while True: try: - usage = callGAPIpages(rep.customerUsageReports(), 'get', 'usageReports', throw_reasons=[gapi.errors.ErrorReason.INVALID], - customerId=customerId, date=tryDate, parameters=parameters) + usage = gapi.get_all_pages(rep.customerUsageReports(), 'get', 'usageReports', throw_reasons=[gapi.errors.ErrorReason.INVALID], + customerId=customerId, date=tryDate, parameters=parameters) break except gapi.errors.GapiInvalidError as e: tryDate = _adjustDate(str(e)) @@ -2140,8 +2039,8 @@ def doPrintAdminRoles(): i += 1 else: controlflow.system_error_exit(2, '%s is not a valid argument for "gam print adminroles".' % sys.argv[i]) - roles = callGAPIpages(cd.roles(), 'list', 'items', - customer=GC_Values[GC_CUSTOMER_ID], fields=fields) + roles = gapi.get_all_pages(cd.roles(), 'list', 'items', + customer=GC_Values[GC_CUSTOMER_ID], fields=fields) for role in roles: role_attrib = {} for key, value in list(role.items()): @@ -2171,8 +2070,8 @@ def doPrintAdmins(): i += 1 else: controlflow.system_error_exit(2, '%s is not a valid argument for "gam print admins".' % sys.argv[i]) - admins = callGAPIpages(cd.roleAssignments(), 'list', 'items', - customer=GC_Values[GC_CUSTOMER_ID], userKey=userKey, roleId=roleId, fields=fields) + admins = gapi.get_all_pages(cd.roleAssignments(), 'list', 'items', + customer=GC_Values[GC_CUSTOMER_ID], userKey=userKey, roleId=roleId, fields=fields) for admin in admins: admin_attrib = {} for key, value in list(admin.items()): @@ -2203,9 +2102,9 @@ def orgunit_from_orgunitid(orgunitid): def buildRoleIdToNameToIdMap(): cd = buildGAPIObject('directory') - result = callGAPIpages(cd.roles(), 'list', 'items', - customer=GC_Values[GC_CUSTOMER_ID], - fields='nextPageToken,items(roleId,roleName)') + result = gapi.get_all_pages(cd.roles(), 'list', 'items', + customer=GC_Values[GC_CUSTOMER_ID], + fields='nextPageToken,items(roleId,roleName)') GM_Globals[GM_MAP_ROLE_ID_TO_NAME] = {} GM_Globals[GM_MAP_ROLE_NAME_TO_ID] = {} for role in result: @@ -2234,9 +2133,9 @@ def getRoleId(role): def buildUserIdToNameMap(): cd = buildGAPIObject('directory') - result = callGAPIpages(cd.users(), 'list', 'users', - customer=GC_Values[GC_CUSTOMER_ID], - fields='nextPageToken,users(id,primaryEmail)') + result = gapi.get_all_pages(cd.users(), 'list', 'users', + customer=GC_Values[GC_CUSTOMER_ID], + fields='nextPageToken,users(id,primaryEmail)') GM_Globals[GM_MAP_USER_ID_TO_NAME] = {} for user in result: GM_Globals[GM_MAP_USER_ID_TO_NAME][user['id']] = user['primaryEmail'] @@ -2250,7 +2149,7 @@ def appID2app(dt, appID): for serviceName, serviceID in list(SERVICE_NAME_TO_ID_MAP.items()): if appID == serviceID: return serviceName - online_services = callGAPIpages(dt.applications(), 'list', 'applications', customerId=GC_Values[GC_CUSTOMER_ID]) + online_services = gapi.get_all_pages(dt.applications(), 'list', 'applications', customerId=GC_Values[GC_CUSTOMER_ID]) for online_service in online_services: if appID == online_service['id']: return online_service['name'] @@ -2260,7 +2159,7 @@ def app2appID(dt, app): serviceName = app.lower() if serviceName in SERVICE_NAME_CHOICES_MAP: return (SERVICE_NAME_CHOICES_MAP[serviceName], SERVICE_NAME_TO_ID_MAP[SERVICE_NAME_CHOICES_MAP[serviceName]]) - online_services = callGAPIpages(dt.applications(), 'list', 'applications', customerId=GC_Values[GC_CUSTOMER_ID]) + online_services = gapi.get_all_pages(dt.applications(), 'list', 'applications', customerId=GC_Values[GC_CUSTOMER_ID]) for online_service in online_services: if serviceName == online_service['name'].lower(): return (online_service['name'], online_service['id']) @@ -2317,7 +2216,7 @@ def doCreateDataTransfer(): def doPrintTransferApps(): dt = buildGAPIObject('datatransfer') - apps = callGAPIpages(dt.applications(), 'list', 'applications', customerId=GC_Values[GC_CUSTOMER_ID]) + apps = gapi.get_all_pages(dt.applications(), 'list', 'applications', customerId=GC_Values[GC_CUSTOMER_ID]) for app in apps: print_json(None, app) print() @@ -2347,9 +2246,9 @@ def doPrintDataTransfers(): i += 1 else: controlflow.system_error_exit(2, '%s is not a valid argument for "gam print transfers"' % sys.argv[i]) - transfers = callGAPIpages(dt.transfers(), 'list', 'dataTransfers', - customerId=GC_Values[GC_CUSTOMER_ID], status=status, - newOwnerUserId=newOwnerUserId, oldOwnerUserId=oldOwnerUserId) + transfers = gapi.get_all_pages(dt.transfers(), 'list', 'dataTransfers', + customerId=GC_Values[GC_CUSTOMER_ID], status=status, + newOwnerUserId=newOwnerUserId, oldOwnerUserId=oldOwnerUserId) for transfer in transfers: for i in range(0, len(transfer['applicationDataTransfers'])): a_transfer = {} @@ -2440,7 +2339,7 @@ def doPrintShowGuardians(csvFormat): sys.stderr.write('\r') sys.stderr.flush() sys.stderr.write('Getting %s for %s%s%s' % (itemName, studentId, currentCount(i, count), ' ' * 40)) - guardians = callGAPIpages(service, 'list', items, soft_errors=True, **kwargs) + guardians = gapi.get_all_pages(service, 'list', items, soft_errors=True, **kwargs) if not csvFormat: print('Student: {0}, {1}:{2}'.format(studentId, itemName, currentCount(i, count))) for guardian in guardians: @@ -2514,10 +2413,10 @@ def doDeleteGuardian(): if not invitationsOnly: if guardianIdIsEmail: try: - results = callGAPIpages(croom.userProfiles().guardians(), 'list', 'guardians', - throw_reasons=[gapi.errors.ErrorReason.FORBIDDEN], - studentId=studentId, invitedEmailAddress=guardianId, - fields='nextPageToken,guardians(studentId,guardianId)') + results = gapi.get_all_pages(croom.userProfiles().guardians(), 'list', 'guardians', + throw_reasons=[gapi.errors.ErrorReason.FORBIDDEN], + studentId=studentId, invitedEmailAddress=guardianId, + fields='nextPageToken,guardians(studentId,guardianId)') if results: for result in results: _deleteGuardian(croom, result['studentId'], result['guardianId'], guardianId) @@ -2531,10 +2430,10 @@ def doDeleteGuardian(): # See if there's a pending invitation if guardianIdIsEmail: try: - results = callGAPIpages(croom.userProfiles().guardianInvitations(), 'list', 'guardianInvitations', - throw_reasons=[gapi.errors.ErrorReason.FORBIDDEN], - studentId=studentId, invitedEmailAddress=guardianId, states=['PENDING',], - fields='nextPageToken,guardianInvitations(studentId,invitationId)') + results = gapi.get_all_pages(croom.userProfiles().guardianInvitations(), 'list', 'guardianInvitations', + throw_reasons=[gapi.errors.ErrorReason.FORBIDDEN], + studentId=studentId, invitedEmailAddress=guardianId, states=['PENDING',], + fields='nextPageToken,guardianInvitations(studentId,invitationId)') if results: for result in results: status = _cancelGuardianInvitation(croom, result['studentId'], result['invitationId']) @@ -2572,10 +2471,10 @@ def doGetCourseInfo(): info = gapi.call(croom.courses(), 'get', id=courseId) info['ownerEmail'] = convertUIDtoEmailAddress('uid:%s' % info['ownerId']) print_json(None, info) - teachers = callGAPIpages(croom.courses().teachers(), 'list', 'teachers', courseId=courseId) - students = callGAPIpages(croom.courses().students(), 'list', 'students', courseId=courseId) + teachers = gapi.get_all_pages(croom.courses().teachers(), 'list', 'teachers', courseId=courseId) + students = gapi.get_all_pages(croom.courses().students(), 'list', 'students', courseId=courseId) try: - aliases = callGAPIpages(croom.courses().aliases(), 'list', 'aliases', throw_reasons=[gapi.errors.ErrorReason.NOT_IMPLEMENTED], courseId=courseId) + aliases = gapi.get_all_pages(croom.courses().aliases(), 'list', 'aliases', throw_reasons=[gapi.errors.ErrorReason.NOT_IMPLEMENTED], courseId=courseId) except gapi.errors.GapiNotImplementedError: aliases = [] if aliases: @@ -2715,7 +2614,7 @@ def _saveParticipants(course, participants, role): fields = 'nextPageToken,courses({0})'.format(','.join(set(fieldsList))) if fieldsList else None printGettingAllItems('Courses', None) page_message = 'Got %%total_items%% Courses...\n' - all_courses = callGAPIpages(croom.courses(), 'list', 'courses', page_message=page_message, teacherId=teacherId, studentId=studentId, courseStates=courseStates, fields=fields) + all_courses = gapi.get_all_pages(croom.courses(), 'list', 'courses', page_message=page_message, teacherId=teacherId, studentId=studentId, courseStates=courseStates, fields=fields) for course in all_courses: if ownerEmails is not None: ownerId = course['ownerId'] @@ -2742,22 +2641,22 @@ def _saveParticipants(course, participants, role): courseId = course['id'] if showAliases: alias_message = ' Got %%%%total_items%%%% Aliases for course %s%s' % (courseId, currentCount(i, count)) - course_aliases = callGAPIpages(croom.courses().aliases(), 'list', 'aliases', - page_message=alias_message, - courseId=courseId) + course_aliases = gapi.get_all_pages(croom.courses().aliases(), 'list', 'aliases', + page_message=alias_message, + courseId=courseId) course['Aliases'] = delimiter.join([alias['alias'][2:] for alias in course_aliases]) if showMembers: if showMembers != 'students': teacher_message = ' Got %%%%total_items%%%% Teachers for course %s%s' % (courseId, currentCount(i, count)) - results = callGAPIpages(croom.courses().teachers(), 'list', 'teachers', - page_message=teacher_message, - courseId=courseId, fields=teachersFields) + results = gapi.get_all_pages(croom.courses().teachers(), 'list', 'teachers', + page_message=teacher_message, + courseId=courseId, fields=teachersFields) _saveParticipants(course, results, 'teachers') if showMembers != 'teachers': student_message = ' Got %%%%total_items%%%% Students for course %s%s' % (courseId, currentCount(i, count)) - results = callGAPIpages(croom.courses().students(), 'list', 'students', - page_message=student_message, - courseId=courseId, fields=studentsFields) + results = gapi.get_all_pages(croom.courses().students(), 'list', 'students', + page_message=student_message, + courseId=courseId, fields=studentsFields) _saveParticipants(course, results, 'students') sortCSVTitles(['id', 'name'], titles) writeCSVfile(csvRows, titles, 'Courses', todrive) @@ -2800,8 +2699,8 @@ def doPrintCourseParticipants(): if not courses: printGettingAllItems('Courses', None) page_message = 'Got %%total_items%% Courses...\n' - all_courses = callGAPIpages(croom.courses(), 'list', 'courses', page_message=page_message, - teacherId=teacherId, studentId=studentId, courseStates=courseStates, fields='nextPageToken,courses(id,name)') + all_courses = gapi.get_all_pages(croom.courses(), 'list', 'courses', page_message=page_message, + teacherId=teacherId, studentId=studentId, courseStates=courseStates, fields='nextPageToken,courses(id,name)') else: all_courses = [] for course in courses: @@ -2813,12 +2712,12 @@ def doPrintCourseParticipants(): courseId = course['id'] if showMembers != 'students': page_message = ' Got %%%%total_items%%%% Teachers for course %s (%s/%s)' % (courseId, i, count) - teachers = callGAPIpages(croom.courses().teachers(), 'list', 'teachers', page_message=page_message, courseId=courseId) + teachers = gapi.get_all_pages(croom.courses().teachers(), 'list', 'teachers', page_message=page_message, courseId=courseId) for teacher in teachers: addRowTitlesToCSVfile(flatten_json(teacher, flattened={'courseId': courseId, 'courseName': course['name'], 'userRole': 'TEACHER'}), csvRows, titles) if showMembers != 'teachers': page_message = ' Got %%%%total_items%%%% Students for course %s (%s/%s)' % (courseId, i, count) - students = callGAPIpages(croom.courses().students(), 'list', 'students', page_message=page_message, courseId=courseId) + students = gapi.get_all_pages(croom.courses().students(), 'list', 'students', page_message=page_message, courseId=courseId) for student in students: addRowTitlesToCSVfile(flatten_json(student, flattened={'courseId': courseId, 'courseName': course['name'], 'userRole': 'STUDENT'}), csvRows, titles) sortCSVTitles(['courseId', 'courseName', 'userRole', 'userId'], titles) @@ -3576,7 +3475,7 @@ def doCalendarPrintShowACLs(csvFormat): i += 1 else: controlflow.system_error_exit(2, '%s is not a valid argument for "gam calendar %s"' % (sys.argv[i], ['showacl', 'printacl'][csvFormat])) - acls = callGAPIpages(cal.acl(), 'list', 'items', calendarId=calendarId) + acls = gapi.get_all_pages(cal.acl(), 'list', 'items', calendarId=calendarId) i = 0 if csvFormat: titles = [] @@ -3704,10 +3603,10 @@ def doCalendarPrintEvents(): else: controlflow.system_error_exit(2, '%s is not a valid argument for "gam calendar printevents"' % sys.argv[i]) page_message = 'Got %%%%total_items%%%% events for %s' % calendarId - results = callGAPIpages(cal.events(), 'list', 'items', page_message=page_message, - calendarId=calendarId, q=q, showDeleted=showDeleted, - showHiddenInvitations=showHiddenInvitations, - timeMin=timeMin, timeMax=timeMax, timeZone=timeZone, updatedMin=updatedMin) + results = gapi.get_all_pages(cal.events(), 'list', 'items', page_message=page_message, + calendarId=calendarId, q=q, showDeleted=showDeleted, + showHiddenInvitations=showHiddenInvitations, + timeMin=timeMin, timeMax=timeMax, timeZone=timeZone, updatedMin=updatedMin) for result in results: row = {'calendarId': calendarId} addRowTitlesToCSVfile(flatten_json(result, flattened=row), csvRows, titles) @@ -4053,7 +3952,7 @@ def printShowCalendars(users, csvFormat): user, cal = buildCalendarGAPIObject(user) if not cal: continue - result = callGAPIpages(cal.calendarList(), 'list', 'items', soft_errors=True) + result = gapi.get_all_pages(cal.calendarList(), 'list', 'items', soft_errors=True) jcount = len(result) if not csvFormat: print('User: {0}, Calendars: ({1}/{2})'.format(user, i, count)) @@ -4081,7 +3980,7 @@ def showCalSettings(users): user, cal = buildCalendarGAPIObject(user) if not cal: continue - feed = callGAPIpages(cal.settings(), 'list', 'items', soft_errors=True) + feed = gapi.get_all_pages(cal.settings(), 'list', 'items', soft_errors=True) if feed: print('User: {0}, Calendar Settings: ({1}/{2})'.format(user, i, count)) settings = {} @@ -4168,10 +4067,10 @@ def printDriveActivity(users): if not activity: continue page_message = 'Got %%%%total_items%%%% activities for %s' % user - feed = callGAPIpages(activity.activities(), 'list', 'activities', - page_message=page_message, source='drive.google.com', userId='me', - drive_ancestorId=drive_ancestorId, groupingStrategy='none', - drive_fileId=drive_fileId) + feed = gapi.get_all_pages(activity.activities(), 'list', 'activities', + page_message=page_message, source='drive.google.com', userId='me', + drive_ancestorId=drive_ancestorId, groupingStrategy='none', + drive_fileId=drive_fileId) for item in feed: addRowTitlesToCSVfile(flatten_json(item['combinedEvent']), csvRows, titles) writeCSVfile(csvRows, titles, 'Drive Activity', todrive) @@ -4206,9 +4105,9 @@ def showDriveFileACL(users): user, drive = buildDrive3GAPIObject(user) if not drive: continue - feed = callGAPIpages(drive.permissions(), 'list', 'permissions', - fileId=fileId, fields='*', supportsAllDrives=True, - useDomainAdminAccess=useDomainAdminAccess) + feed = gapi.get_all_pages(drive.permissions(), 'list', 'permissions', + fileId=fileId, fields='*', supportsAllDrives=True, + useDomainAdminAccess=useDomainAdminAccess) for permission in feed: printPermission(permission) print('') @@ -4449,9 +4348,9 @@ def printDriveFileList(users): continue sys.stderr.write('Getting files for %s...\n' % user) page_message = ' Got %%%%total_items%%%% files for %s...\n' % user - feed = callGAPIpages(drive.files(), 'list', 'items', - page_message=page_message, soft_errors=True, - q=query, orderBy=orderBy, fields=fields) + feed = gapi.get_all_pages(drive.files(), 'list', 'items', + page_message=page_message, soft_errors=True, + q=query, orderBy=orderBy, fields=fields) for f_file in feed: a_file = {'Owner': user} for attrib in f_file: @@ -4503,9 +4402,9 @@ def doDriveSearch(drive, query=None, quiet=False): page_message = ' Got %%total_items%% files...\n' else: page_message = None - files = callGAPIpages(drive.files(), 'list', 'items', - page_message=page_message, - q=query, fields='nextPageToken,items(id)') + files = gapi.get_all_pages(drive.files(), 'list', 'items', + page_message=page_message, + q=query, fields='nextPageToken,items(id)') ids = list() for f_file in files: ids.append(f_file['id']) @@ -4624,8 +4523,8 @@ def showDriveFileTree(users): root_folder = gapi.call(drive.about(), 'get', fields='rootFolderId')['rootFolderId'] sys.stderr.write('Getting all files for %s...\n' % user) page_message = ' Got %%%%total_items%%%% files for %s...\n' % user - feed = callGAPIpages(drive.files(), 'list', 'items', page_message=page_message, - q=query, orderBy=orderBy, fields='items(id,title,parents(id),mimeType),nextPageToken') + feed = gapi.get_all_pages(drive.files(), 'list', 'items', page_message=page_message, + q=query, orderBy=orderBy, fields='items(id,title,parents(id),mimeType),nextPageToken') printDriveFolderContents(feed, root_folder, 0) def deleteEmptyDriveFolders(users): @@ -4638,8 +4537,8 @@ def deleteEmptyDriveFolders(users): while deleted_empty: sys.stderr.write('Getting folders for %s...\n' % user) page_message = ' Got %%%%total_items%%%% folders for %s...\n' % user - feed = callGAPIpages(drive.files(), 'list', 'items', page_message=page_message, - q=query, fields='items(title,id),nextPageToken') + feed = gapi.get_all_pages(drive.files(), 'list', 'items', page_message=page_message, + q=query, fields='items(title,id),nextPageToken') deleted_empty = False for folder in feed: children = gapi.call(drive.children(), 'list', @@ -5114,8 +5013,8 @@ def transferSecCals(users): user, source_cal = buildCalendarGAPIObject(user) if not source_cal: continue - calendars = callGAPIpages(source_cal.calendarList(), 'list', 'items', soft_errors=True, - minAccessRole='owner', showHidden=True, fields='items(id),nextPageToken') + calendars = gapi.get_all_pages(source_cal.calendarList(), 'list', 'items', soft_errors=True, + minAccessRole='owner', showHidden=True, fields='items(id),nextPageToken') for calendar in calendars: calendarId = calendar['id'] if calendarId.find('@group.calendar.google.com') != -1: @@ -5162,16 +5061,16 @@ def transferDriveFiles(users): source_permissionid = source_about['permissionId'] print("Getting file list for source user: %s..." % user) page_message = ' Got %%total_items%% files\n' - source_drive_files = callGAPIpages(source_drive.files(), 'list', 'items', page_message=page_message, - q="'me' in owners and trashed = false", fields='items(id,parents,mimeType),nextPageToken') + source_drive_files = gapi.get_all_pages(source_drive.files(), 'list', 'items', page_message=page_message, + q="'me' in owners and trashed = false", fields='items(id,parents,mimeType),nextPageToken') all_source_file_ids = [] for source_drive_file in source_drive_files: all_source_file_ids.append(source_drive_file['id']) total_count = len(source_drive_files) print("Getting folder list for target user: %s..." % target_user) page_message = ' Got %%total_items%% folders\n' - target_folders = callGAPIpages(target_drive.files(), 'list', 'items', page_message=page_message, - q="'me' in owners and mimeType = 'application/vnd.google-apps.folder'", fields='items(id,title),nextPageToken') + target_folders = gapi.get_all_pages(target_drive.files(), 'list', 'items', page_message=page_message, + q="'me' in owners and mimeType = 'application/vnd.google-apps.folder'", fields='items(id,title),nextPageToken') got_top_folder = False all_target_folder_ids = [] for target_folder in target_folders: @@ -5991,7 +5890,7 @@ def doProcessMessagesOrThreads(users, function, unit='messages'): print('Searching %s for %s' % (unit, user)) unitmethod = getattr(gmail.users(), unit) page_message = 'Got %%%%total_items%%%% %s for user %s' % (unit, user) - listResult = callGAPIpages(unitmethod(), 'list', unit, page_message=page_message, + listResult = gapi.get_all_pages(unitmethod(), 'list', unit, page_message=page_message, userId='me', q=query, includeSpamTrash=True, soft_errors=True, fields='nextPageToken,{0}(id)'.format(unit)) result_count = len(listResult) if not doIt or result_count == 0: @@ -6216,8 +6115,8 @@ def renameLabels(users): except gapi.errors.gapi.errors.GapiAbortedError: if merge: print(' Merging %s label to existing %s label' % (label['name'], new_label_name)) - messages_to_relabel = callGAPIpages(gmail.users().messages(), 'list', 'messages', - userId=user, q='label:%s' % cleanLabelQuery(label['name'])) + messages_to_relabel = gapi.get_all_pages(gmail.users().messages(), 'list', 'messages', + userId=user, q='label:%s' % cleanLabelQuery(label['name'])) if messages_to_relabel: for new_label in labels['labels']: if new_label['name'].lower() == new_label_name.lower(): @@ -7564,7 +7463,7 @@ def enableGAMProjectAPIs(GAMProjectAPIs, httpObj, projectId, checkEnabled, i=0, status = True if checkEnabled: try: - services = callGAPIpages(serveman.services(), 'list', 'services', + services = gapi.get_all_pages(serveman.services(), 'list', 'services', throw_reasons=[gapi.errors.ErrorReason.NOT_FOUND], consumerId=project_name, fields='nextPageToken,services(serviceName)') jcount = len(services) @@ -7721,7 +7620,7 @@ def _getCurrentProjectID(): def _getProjects(crm, pfilter): try: - return callGAPIpages(crm.projects(), 'list', 'projects', throw_reasons=[gapi.errors.ErrorReason.BAD_REQUEST], filter=pfilter) + return gapi.get_all_pages(crm.projects(), 'list', 'projects', throw_reasons=[gapi.errors.ErrorReason.BAD_REQUEST], filter=pfilter) except gapi.errors.GapiBadRequestError as e: controlflow.system_error_exit(2, 'Project: {0}, {1}'.format(pfilter, str(e))) @@ -7791,7 +7690,7 @@ def convertGCPFolderNameToID(parent, crm2): # crm2.folders() is broken requiring pageToken, etc in body, not URL. # for now just use callGAPI and if user has that many folders they'll # just need to be specific. - folders = gapi.get_first_page( + folders = gapi.get_items( crm2.folders(), 'search', items='folders', body={'pageSize': 1000, 'query': 'displayName="%s"' % parent}) if not folders: @@ -8111,9 +8010,9 @@ def printShowTeamDrives(users, csvFormat): user, drive = buildDrive3GAPIObject(user) if not drive: continue - results = callGAPIpages(drive.drives(), 'list', 'drives', - useDomainAdminAccess=useDomainAdminAccess, fields='*', - q=q, soft_errors=True) + results = gapi.get_all_pages(drive.drives(), 'list', 'drives', + useDomainAdminAccess=useDomainAdminAccess, fields='*', + q=q, soft_errors=True) if not results: continue for td in results: @@ -8384,7 +8283,7 @@ def doDownloadCloudStorageBucket(): controlflow.system_error_exit(5, 'Could not find a takeout-export-* bucket in that URL') s = buildGAPIObject('storage') page_message = 'Got %%total_items%% files...' - objects = callGAPIpages(s.objects(), 'list', 'items', page_message=page_message, bucket=bucket, projection='noAcl', fields='nextPageToken,items(name,id,md5Hash)') + objects = gapi.get_all_pages(s.objects(), 'list', 'items', page_message=page_message, bucket=bucket, projection='noAcl', fields='nextPageToken,items(name,id,md5Hash)') i = 1 for object_ in objects: print("%s/%s" % (i, len(objects))) @@ -8582,7 +8481,7 @@ def convertExportNameToID(v, nameOrID, matterId): cg = UID_PATTERN.match(nameOrID) if cg: return cg.group(1) - exports = callGAPIpages(v.matters().exports(), 'list', 'exports', matterId=matterId, fields='exports(id,name),nextPageToken') + exports = gapi.get_all_pages(v.matters().exports(), 'list', 'exports', matterId=matterId, fields='exports(id,name),nextPageToken') for export in exports: if export['name'].lower() == nameOrID: return export['id'] @@ -8593,7 +8492,7 @@ def convertHoldNameToID(v, nameOrID, matterId): cg = UID_PATTERN.match(nameOrID) if cg: return cg.group(1) - holds = callGAPIpages(v.matters().holds(), 'list', 'holds', matterId=matterId, fields='holds(holdId,name),nextPageToken') + holds = gapi.get_all_pages(v.matters().holds(), 'list', 'holds', matterId=matterId, fields='holds(holdId,name),nextPageToken') for hold in holds: if hold['name'].lower() == nameOrID: return hold['holdId'] @@ -8604,7 +8503,7 @@ def convertMatterNameToID(v, nameOrID): cg = UID_PATTERN.match(nameOrID) if cg: return cg.group(1) - matters = callGAPIpages(v.matters(), 'list', 'matters', view='BASIC', fields='matters(matterId,name),nextPageToken') + matters = gapi.get_all_pages(v.matters(), 'list', 'matters', view='BASIC', fields='matters(matterId,name),nextPageToken') for matter in matters: if matter['name'].lower() == nameOrID: return matter['matterId'] @@ -8954,9 +8853,9 @@ def doCreateBuilding(): customer=GC_Values[GC_CUSTOMER_ID], body=body) def _makeBuildingIdNameMap(cd): - buildings = callGAPIpages(cd.resources().buildings(), 'list', 'buildings', - customer=GC_Values[GC_CUSTOMER_ID], - fields='nextPageToken,buildings(buildingId,buildingName)') + buildings = gapi.get_all_pages(cd.resources().buildings(), 'list', 'buildings', + customer=GC_Values[GC_CUSTOMER_ID], + fields='nextPageToken,buildings(buildingId,buildingName)') GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] = {} GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] = {} for building in buildings: @@ -9174,7 +9073,7 @@ def doRemoveUsersAliases(users): def deleteUserFromGroups(users): cd = buildGAPIObject('directory') for user in users: - user_groups = callGAPIpages(cd.groups(), 'list', 'groups', userKey=user, fields='groups(id,email)') + user_groups = gapi.get_all_pages(cd.groups(), 'list', 'groups', userKey=user, fields='groups(id,email)') num_groups = len(user_groups) print('%s is in %s groups' % (user, num_groups)) j = 0 @@ -9401,10 +9300,10 @@ def _getRoleAndUsers(): page_message = 'Got %%%%total_items%%%% %s...' % member_type_message validRoles, listRoles, listFields = _getRoleVerification(roles, 'nextPageToken,members({0})'.format(','.join(fields))) try: - result = callGAPIpages(cd.members(), 'list', 'members', - page_message=page_message, - throw_reasons=gapi.errors.MEMBERS_THROW_REASONS, - groupKey=group, roles=listRoles, fields=listFields) + result = gapi.get_all_pages(cd.members(), 'list', 'members', + page_message=page_message, + throw_reasons=gapi.errors.MEMBERS_THROW_REASONS, + groupKey=group, roles=listRoles, fields=listFields) if not result: print('Group already has 0 members') return @@ -9499,9 +9398,9 @@ def getCrOSDeviceEntity(i, cd): kwargs = {'orgUnitPath': query[12:]} else: kwargs = {'query': query} - devices = callGAPIpages(cd.chromeosdevices(), 'list', 'chromeosdevices', - customerId=GC_Values[GC_CUSTOMER_ID], - fields='nextPageToken,chromeosdevices(deviceId)', **kwargs) + devices = gapi.get_all_pages(cd.chromeosdevices(), 'list', 'chromeosdevices', + customerId=GC_Values[GC_CUSTOMER_ID], + fields='nextPageToken,chromeosdevices(deviceId)', **kwargs) return i+1, [device['deviceId'] for device in devices] return i+1, sys.argv[i].replace(',', ' ').split() @@ -9585,10 +9484,10 @@ def doUpdateMobile(): query = resourceIds[6:] fields = 'nextPageToken,mobiledevices(resourceId,email)' page_message = 'Got %%total_items%% mobile devices...\n' - devices = callGAPIpages(cd.mobiledevices(), 'list', - page_message=page_message, - customerId=GC_Values[GC_CUSTOMER_ID], - items='mobiledevices', query=query, fields=fields) + devices = gapi.get_all_pages(cd.mobiledevices(), 'list', + page_message=page_message, + customerId=GC_Values[GC_CUSTOMER_ID], + items='mobiledevices', query=query, fields=fields) else: devices = [{'resourceId': resourceIds, 'email': ['not set']}] doit = True @@ -10151,7 +10050,7 @@ def user_lic_result(request_id, response, exception): for alias in user['nonEditableAliases']: print(' %s' % alias) if getGroups: - groups = callGAPIpages(cd.groups(), 'list', 'groups', userKey=user_email, fields='groups(name,email),nextPageToken') + groups = gapi.get_all_pages(cd.groups(), 'list', 'groups', userKey=user_email, fields='groups(name,email),nextPageToken') if groups: print('Groups: (%s)' % len(groups)) for group in groups: @@ -10230,14 +10129,14 @@ def doGetGroupInfo(group_name=None): continue print(' %s: %s' % (key, value)) if getGroups: - groups = callGAPIpages(cd.groups(), 'list', 'groups', - userKey=basic_info['email'], fields='nextPageToken,groups(name,email)') + groups = gapi.get_all_pages(cd.groups(), 'list', 'groups', + userKey=basic_info['email'], fields='nextPageToken,groups(name,email)') if groups: print('Groups: ({0})'.format(len(groups))) for groupm in groups: print(' %s: %s' % (groupm['name'], groupm['email'])) if getUsers: - members = callGAPIpages(cd.members(), 'list', 'members', groupKey=group_name, fields='nextPageToken,members(email,id,role,type)') + members = gapi.get_all_pages(cd.members(), 'list', 'members', groupKey=group_name, fields='nextPageToken,members(email,id,role,type)') print('Members:') for member in members: print(' %s: %s (%s)' % (member.get('role', ROLE_MEMBER).lower(), member.get('email', member['id']), member['type'].lower())) @@ -10524,7 +10423,7 @@ def doSiteVerifyShow(): def doGetSiteVerifications(): verif = buildGAPIObject('siteVerification') - sites = gapi.get_first_page(verif.webResource(), 'list', 'items') + sites = gapi.get_items(verif.webResource(), 'list', 'items') if sites: for site in sites: print('Site: %s' % site['site']['identifier']) @@ -10717,9 +10616,9 @@ def doGetOrgInfo(name=None, return_attrib=None): if get_users: name = result['orgUnitPath'] page_message = 'Got %%total_items%% Users: %%first_item%% - %%last_item%%\n' - users = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, - message_attribute='primaryEmail', customer=GC_Values[GC_CUSTOMER_ID], query=orgUnitPathQuery(name, checkSuspended), - fields='users(primaryEmail,orgUnitPath),nextPageToken') + users = gapi.get_all_pages(cd.users(), 'list', 'users', page_message=page_message, + message_attribute='primaryEmail', customer=GC_Values[GC_CUSTOMER_ID], query=orgUnitPathQuery(name, checkSuspended), + fields='users(primaryEmail,orgUnitPath),nextPageToken') if checkSuspended is None: print('Users:') elif not checkSuspended: @@ -10737,7 +10636,7 @@ def doGetOrgInfo(name=None, return_attrib=None): def doGetASPs(users): cd = buildGAPIObject('directory') for user in users: - asps = gapi.get_first_page(cd.asps(), 'list', 'items', userKey=user) + asps = gapi.get_items(cd.asps(), 'list', 'items', userKey=user) if asps: print('Application-Specific Passwords for %s' % user) for asp in asps: @@ -10763,7 +10662,7 @@ def doDelASP(users): codeIds = codeIdList.replace(',', ' ').split() for user in users: if allCodeIds: - asps = gapi.get_first_page(cd.asps(), 'list', 'items', userKey=user, fields='items/codeId') + asps = gapi.get_items(cd.asps(), 'list', 'items', userKey=user, fields='items/codeId') codeIds = [asp['codeId'] for asp in asps] for codeId in codeIds: gapi.call(cd.asps(), 'delete', userKey=user, codeId=codeId) @@ -10788,7 +10687,7 @@ def doGetBackupCodes(users): cd = buildGAPIObject('directory') for user in users: try: - codes = gapi.get_first_page(cd.verificationCodes(), 'list', 'items', throw_reasons=[gapi.errors.ErrorReason.INVALID_ARGUMENT, gapi.errors.ErrorReason.INVALID], userKey=user) + codes = gapi.get_items(cd.verificationCodes(), 'list', 'items', throw_reasons=[gapi.errors.ErrorReason.INVALID_ARGUMENT, gapi.errors.ErrorReason.INVALID], userKey=user) except (gapi.errors.GapiInvalidArgumentError, gapi.errors.GapiInvalidError): codes = [] printBackupCodes(user, codes) @@ -10797,7 +10696,7 @@ def doGenBackupCodes(users): cd = buildGAPIObject('directory') for user in users: gapi.call(cd.verificationCodes(), 'generate', userKey=user) - codes = gapi.get_first_page(cd.verificationCodes(), 'list', 'items', userKey=user) + codes = gapi.get_items(cd.verificationCodes(), 'list', 'items', userKey=user) printBackupCodes(user, codes) def doDelBackupCodes(users): @@ -10883,9 +10782,9 @@ def _showToken(token): throw_reasons=[gapi.errors.ErrorReason.NOT_FOUND, gapi.errors.ErrorReason.USER_NOT_FOUND, gapi.errors.ErrorReason.RESOURCE_NOT_FOUND], userKey=user, clientId=clientId, fields=fields)] else: - results = gapi.get_first_page(cd.tokens(), 'list', 'items', - throw_reasons=[gapi.errors.ErrorReason.USER_NOT_FOUND], - userKey=user, fields='items({0})'.format(fields)) + results = gapi.get_items(cd.tokens(), 'list', 'items', + throw_reasons=[gapi.errors.ErrorReason.USER_NOT_FOUND], + userKey=user, fields='items({0})'.format(fields)) jcount = len(results) if not csvFormat: print('User: {0}, Access Tokens ({1}/{2})'.format(user, i, count)) @@ -10911,7 +10810,7 @@ def doDeprovUser(users): cd = buildGAPIObject('directory') for user in users: print('Getting Application Specific Passwords for %s' % user) - asps = gapi.get_first_page(cd.asps(), 'list', 'items', userKey=user, fields='items/codeId') + asps = gapi.get_items(cd.asps(), 'list', 'items', userKey=user, fields='items/codeId') jcount = len(asps) if jcount > 0: j = 0 @@ -10927,7 +10826,7 @@ def doDeprovUser(users): except gapi.errors.GapiInvalidError: print('No 2SV Backup Codes') print('Getting tokens for %s...' % user) - tokens = gapi.get_first_page(cd.tokens(), 'list', 'items', userKey=user, fields='items/clientId') + tokens = gapi.get_items(cd.tokens(), 'list', 'items', userKey=user, fields='items/clientId') jcount = len(tokens) if jcount > 0: j = 0 @@ -10961,8 +10860,8 @@ def doUndeleteUser(): user_uid = user else: print('Looking up UID for %s...' % user) - deleted_users = callGAPIpages(cd.users(), 'list', 'users', - customer=GC_Values[GC_CUSTOMER_ID], showDeleted=True) + deleted_users = gapi.get_all_pages(cd.users(), 'list', 'users', + customer=GC_Values[GC_CUSTOMER_ID], showDeleted=True) matching_users = list() for deleted_user in deleted_users: if str(deleted_user['primaryEmail']).lower() == user: @@ -11453,7 +11352,7 @@ def doPrintUsers(): for query in queries: printGettingAllItems('Users', query) page_message = 'Got %%total_items%% Users: %%first_item%% - %%last_item%%\n' - all_users = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, + all_users = gapi.get_all_pages(cd.users(), 'list', 'users', page_message=page_message, message_attribute='primaryEmail', customer=customer, domain=domain, fields=fields, showDeleted=deleted_only, orderBy=orderBy, sortOrder=sortOrder, viewType=viewType, query=query, projection=projection, customFieldMask=customFieldMask) @@ -11472,7 +11371,7 @@ def doPrintUsers(): for user in csvRows: user_email = user['primaryEmail'] sys.stderr.write("Getting Group Membership for %s (%s/%s)\r\n" % (user_email, user_count, total_users)) - groups = callGAPIpages(cd.groups(), 'list', 'groups', userKey=user_email) + groups = gapi.get_all_pages(cd.groups(), 'list', 'groups', userKey=user_email) user['Groups'] = groupDelimiter.join([groupname['email'] for groupname in groups]) user_count += 1 if getLicenseFeed: @@ -11487,7 +11386,7 @@ def doPrintUsers(): def doPrintShowAlerts(): _, ac = buildAlertCenterGAPIObject(_getValueFromOAuth('email')) - alerts = callGAPIpages(ac.alerts(), 'list', 'alerts') + alerts = gapi.get_all_pages(ac.alerts(), 'list', 'alerts') titles = [] csv_rows = [] for alert in alerts: @@ -11500,7 +11399,7 @@ def doPrintShowAlerts(): def doPrintShowAlertFeedback(): _, ac = buildAlertCenterGAPIObject(_getValueFromOAuth('email')) - feedback = callGAPIpages(ac.alerts().feedback(), 'list', 'feedback', alertId='-') + feedback = gapi.get_all_pages(ac.alerts().feedback(), 'list', 'feedback', alertId='-') for feedbac in feedback: print(feedbac) @@ -11705,10 +11604,10 @@ def doPrintGroups(): addTitlesToCSVfile(['Owners',], titles) printGettingAllItems('Groups', None) page_message = 'Got %%total_items%% Groups: %%first_item%% - %%last_item%%\n' - entityList = callGAPIpages(cd.groups(), 'list', 'groups', - page_message=page_message, message_attribute='email', - customer=customer, domain=usedomain, userKey=usemember, query=usequery, - fields='nextPageToken,groups({0})'.format(cdfields)) + entityList = gapi.get_all_pages(cd.groups(), 'list', 'groups', + page_message=page_message, message_attribute='email', + customer=customer, domain=usedomain, userKey=usemember, query=usequery, + fields='nextPageToken,groups({0})'.format(cdfields)) i = 0 count = len(entityList) for groupEntity in entityList: @@ -11725,10 +11624,10 @@ def doPrintGroups(): sys.stderr.write(' Getting %s for %s (%s/%s)\n' % (roles, groupEmail, i, count)) page_message = ' Got %%total_items%% members: %%first_item%% - %%last_item%%\n' validRoles, listRoles, listFields = _getRoleVerification(roles, 'nextPageToken,members(email,id,role)') - groupMembers = callGAPIpages(cd.members(), 'list', 'members', - page_message=page_message, message_attribute='email', - soft_errors=True, - groupKey=groupEmail, roles=listRoles, fields=listFields) + groupMembers = gapi.get_all_pages(cd.members(), 'list', 'members', + page_message=page_message, message_attribute='email', + soft_errors=True, + groupKey=groupEmail, roles=listRoles, fields=listFields) if members: membersList = [] membersCount = 0 @@ -11913,7 +11812,7 @@ def doPrintAliases(): for query in queries: printGettingAllItems('User Aliases', query) page_message = 'Got %%total_items%% Users %%first_item%% - %%last_item%%\n' - all_users = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, + all_users = gapi.get_all_pages(cd.users(), 'list', 'users', page_message=page_message, message_attribute='primaryEmail', customer=GC_Values[GC_CUSTOMER_ID], query=query, fields='nextPageToken,users({0})'.format(','.join(userFields))) for user in all_users: @@ -11924,7 +11823,7 @@ def doPrintAliases(): if doGroups: printGettingAllItems('Group Aliases', None) page_message = 'Got %%total_items%% Groups %%first_item%% - %%last_item%%\n' - all_groups = callGAPIpages(cd.groups(), 'list', 'groups', page_message=page_message, + all_groups = gapi.get_all_pages(cd.groups(), 'list', 'groups', page_message=page_message, message_attribute='email', customer=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,groups({0})'.format(','.join(groupFields))) for group in all_groups: @@ -11996,9 +11895,9 @@ def doPrintGroupMembers(): else: controlflow.system_error_exit(2, '%s is not a valid argument for "gam print group-members"' % sys.argv[i]) if not groups_to_get: - groups_to_get = callGAPIpages(cd.groups(), 'list', 'groups', message_attribute='email', - customer=customer, domain=usedomain, userKey=usemember, query=usequery, - fields='nextPageToken,groups(email)') + groups_to_get = gapi.get_all_pages(cd.groups(), 'list', 'groups', message_attribute='email', + customer=customer, domain=usedomain, userKey=usemember, query=usequery, + fields='nextPageToken,groups(email)') i = 0 count = len(groups_to_get) for group in groups_to_get: @@ -12006,7 +11905,7 @@ def doPrintGroupMembers(): group_email = group['email'] sys.stderr.write('Getting members for %s (%s/%s)\n' % (group_email, i, count)) validRoles, listRoles, listFields = _getRoleVerification(','.join(roles), fields) - group_members = callGAPIpages(cd.members(), 'list', 'members', + group_members = gapi.get_all_pages(cd.members(), 'list', 'members', soft_errors=True, includeDerivedMembership=includeDerivedMembership, groupKey=group_email, roles=listRoles, fields=listFields) @@ -12068,7 +11967,7 @@ def doPrintVaultMatters(): controlflow.system_error_exit(3, '%s is not a valid argument to "gam print matters"' % myarg) printGettingAllItems('Vault Matters', None) page_message = 'Got %%total_items%% Vault Matters...\n' - matters = callGAPIpages(v.matters(), 'list', 'matters', page_message=page_message, view=view) + matters = gapi.get_all_pages(v.matters(), 'list', 'matters', page_message=page_message, view=view) for matter in matters: addRowTitlesToCSVfile(flatten_json(matter), csvRows, titles) sortCSVTitles(initialTitles, titles) @@ -12094,7 +11993,7 @@ def doPrintVaultExports(): else: controlflow.system_error_exit(3, '%s is not a valid a valid argument to "gam print exports"' % myarg) if not matters: - matters_results = callGAPIpages(v.matters(), 'list', 'matters', view='BASIC', fields='matters(matterId,state),nextPageToken') + matters_results = gapi.get_all_pages(v.matters(), 'list', 'matters', view='BASIC', fields='matters(matterId,state),nextPageToken') for matter in matters_results: if matter['state'] != 'OPEN': print('ignoring matter %s in state %s' % (matter['matterId'], matter['state'])) @@ -12105,7 +12004,7 @@ def doPrintVaultExports(): matterIds.append(getMatterItem(v, matter)) for matterId in matterIds: sys.stderr.write('Retrieving exports for matter %s\n' % matterId) - exports = callGAPIpages(v.matters().exports(), 'list', 'exports', matterId=matterId) + exports = gapi.get_all_pages(v.matters().exports(), 'list', 'exports', matterId=matterId) for export in exports: addRowTitlesToCSVfile(flatten_json(export, flattened={'matterId': matterId}), csvRows, titles) sortCSVTitles(initialTitles, titles) @@ -12131,7 +12030,7 @@ def doPrintVaultHolds(): else: controlflow.system_error_exit(3, '%s is not a valid a valid argument to "gam print holds"' % myarg) if not matters: - matters_results = callGAPIpages(v.matters(), 'list', 'matters', view='BASIC', fields='matters(matterId,state),nextPageToken') + matters_results = gapi.get_all_pages(v.matters(), 'list', 'matters', view='BASIC', fields='matters(matterId,state),nextPageToken') for matter in matters_results: if matter['state'] != 'OPEN': print('ignoring matter %s in state %s' % (matter['matterId'], matter['state'])) @@ -12142,7 +12041,7 @@ def doPrintVaultHolds(): matterIds.append(getMatterItem(v, matter)) for matterId in matterIds: sys.stderr.write('Retrieving holds for matter %s\n' % matterId) - holds = callGAPIpages(v.matters().holds(), 'list', 'holds', matterId=matterId) + holds = gapi.get_all_pages(v.matters().holds(), 'list', 'holds', matterId=matterId) for hold in holds: addRowTitlesToCSVfile(flatten_json(hold, flattened={'matterId': matterId}), csvRows, titles) sortCSVTitles(initialTitles, titles) @@ -12201,9 +12100,9 @@ def doPrintMobileDevices(): for query in queries: printGettingAllItems('Mobile Devices', query) page_message = 'Got %%total_items%% Mobile Devices...\n' - all_mobile = callGAPIpages(cd.mobiledevices(), 'list', 'mobiledevices', page_message=page_message, - customerId=GC_Values[GC_CUSTOMER_ID], query=query, projection=projection, fields=fields, - orderBy=orderBy, sortOrder=sortOrder) + all_mobile = gapi.get_all_pages(cd.mobiledevices(), 'list', 'mobiledevices', page_message=page_message, + customerId=GC_Values[GC_CUSTOMER_ID], query=query, projection=projection, fields=fields, + orderBy=orderBy, sortOrder=sortOrder) for mobile in all_mobile: row = {} for attrib in mobile: @@ -12318,7 +12217,7 @@ def doPrintCrosActivity(): for query in queries: printGettingAllItems('CrOS Devices', query) page_message = 'Got %%total_items%% CrOS Devices...\n' - all_cros = callGAPIpages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, + all_cros = gapi.get_all_pages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, query=query, customerId=GC_Values[GC_CUSTOMER_ID], projection='FULL', fields=fields, orgUnitPath=orgUnitPath) for cros in all_cros: @@ -12499,7 +12398,7 @@ def _getSelectedLists(myarg): for query in queries: printGettingAllItems('CrOS Devices', query) page_message = 'Got %%total_items%% CrOS Devices...\n' - all_cros = callGAPIpages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, + all_cros = gapi.get_all_pages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, query=query, customerId=GC_Values[GC_CUSTOMER_ID], projection=projection, orgUnitPath=orgUnitPath, orderBy=orderBy, sortOrder=sortOrder, fields=fields) for cros in all_cros: @@ -12630,8 +12529,8 @@ def doPrintLicenses(returnFields=None, skus=None, countsOnly=False, returnCounts product = products[0] page_message = 'Got %%%%total_items%%%% Licenses for %s...\n' % SKUS.get(sku, {'displayName': sku})['displayName'] try: - licenses += callGAPIpages(lic.licenseAssignments(), 'listForProductAndSku', 'items', throw_reasons=[gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.FORBIDDEN], page_message=page_message, - customerId=GC_Values[GC_DOMAIN], productId=product, skuId=sku, fields=fields) + licenses += gapi.get_all_pages(lic.licenseAssignments(), 'listForProductAndSku', 'items', throw_reasons=[gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.FORBIDDEN], page_message=page_message, + customerId=GC_Values[GC_DOMAIN], productId=product, skuId=sku, fields=fields) if countsOnly: licenseCounts.append(['Product', product, 'SKU', sku, 'Licenses', len(licenses)]) licenses = [] @@ -12643,8 +12542,8 @@ def doPrintLicenses(returnFields=None, skus=None, countsOnly=False, returnCounts for productId in products: page_message = 'Got %%%%total_items%%%% Licenses for %s...\n' % PRODUCTID_NAME_MAPPINGS.get(productId, productId) try: - licenses += callGAPIpages(lic.licenseAssignments(), 'listForProduct', 'items', throw_reasons=[gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.FORBIDDEN], page_message=page_message, - customerId=GC_Values[GC_DOMAIN], productId=productId, fields=fields) + licenses += gapi.get_all_pages(lic.licenseAssignments(), 'listForProduct', 'items', throw_reasons=[gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.FORBIDDEN], page_message=page_message, + customerId=GC_Values[GC_DOMAIN], productId=productId, fields=fields) if countsOnly: licenseCounts.append(['Product', productId, 'Licenses', len(licenses)]) licenses = [] @@ -12745,8 +12644,8 @@ def doPrintFeatures(): controlflow.system_error_exit(3, '%s is not a valid argument to "gam print features"' % sys.argv[i]) if fields: fields = fields % ','.join(fieldsList) - features = callGAPIpages(cd.resources().features(), 'list', 'features', - customer=GC_Values[GC_CUSTOMER_ID], fields=fields) + features = gapi.get_all_pages(cd.resources().features(), 'list', 'features', + customer=GC_Values[GC_CUSTOMER_ID], fields=fields) for feature in features: feature.pop('etags', None) feature.pop('etag', None) @@ -12792,8 +12691,8 @@ def doPrintBuildings(): controlflow.system_error_exit(3, '%s is not a valid argument to "gam print buildings"' % sys.argv[i]) if fields: fields = fields % ','.join(fieldsList) - buildings = callGAPIpages(cd.resources().buildings(), 'list', 'buildings', - customer=GC_Values[GC_CUSTOMER_ID], fields=fields) + buildings = gapi.get_all_pages(cd.resources().buildings(), 'list', 'buildings', + customer=GC_Values[GC_CUSTOMER_ID], fields=fields) for building in buildings: building.pop('etags', None) building.pop('etag', None) @@ -12847,10 +12746,10 @@ def doPrintResourceCalendars(): addFieldToCSVfile('buildingName', {'buildingName': ['buildingName',]}, fieldsList, fieldsTitles, titles) printGettingAllItems('Resource Calendars', None) page_message = 'Got %%total_items%% Resource Calendars: %%first_item%% - %%last_item%%\n' - resources = callGAPIpages(cd.resources().calendars(), 'list', 'items', - page_message=page_message, message_attribute='resourceId', - customer=GC_Values[GC_CUSTOMER_ID], query=query, - fields=fields) + resources = gapi.get_all_pages(cd.resources().calendars(), 'list', 'items', + page_message=page_message, message_attribute='resourceId', + customer=GC_Values[GC_CUSTOMER_ID], query=query, + fields=fields) for resource in resources: if 'featureInstances' in resource: resource['featureInstances'] = ','.join([a_feature['feature']['name'] for a_feature in resource.pop('featureInstances')]) @@ -12917,8 +12816,8 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No sys.stderr.write("Getting %s of %s (may take some time for large groups)...\n" % (member_type_message, group)) page_message = 'Got %%%%total_items%%%% %s...' % member_type_message validRoles, listRoles, listFields = _getRoleVerification(member_type, 'nextPageToken,members(email,id,type,status)') - members = callGAPIpages(cd.members(), 'list', 'members', page_message=page_message, - groupKey=group, roles=listRoles, fields=listFields) + members = gapi.get_all_pages(cd.members(), 'list', 'members', page_message=page_message, + groupKey=group, roles=listRoles, fields=listFields) users = [] for member in members: if ((not groupUserMembersOnly) or (member['type'] == 'USER')) and _checkMemberRoleIsSuspended(member, validRoles, checkSuspended): @@ -12939,7 +12838,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No if not silent: printGettingAllItems('Users', query) page_message = 'Got %%total_items%% Users...' - members = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, + members = gapi.get_all_pages(cd.users(), 'list', 'users', page_message=page_message, customer=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,users(primaryEmail,orgUnitPath)', query=query) ou = ou.lower() @@ -12961,7 +12860,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No if not silent: printGettingAllItems('Users', query) page_message = 'Got %%total_items%% Users...' - members = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, + members = gapi.get_all_pages(cd.users(), 'list', 'users', page_message=page_message, customer=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,users(primaryEmail)', query=query) for member in members: @@ -12980,7 +12879,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No if not silent: printGettingAllItems('Users', query) page_message = 'Got %%total_items%% Users...' - members = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, + members = gapi.get_all_pages(cd.users(), 'list', 'users', page_message=page_message, customer=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,users(primaryEmail,suspended)', query=query) for member in members: @@ -13025,14 +12924,14 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No entity = addCourseIdScope(entity) if entity_type in ['courseparticipants', 'teachers']: page_message = 'Got %%total_items%% Teachers...' - teachers = callGAPIpages(croom.courses().teachers(), 'list', 'teachers', page_message=page_message, courseId=entity) + teachers = gapi.get_all_pages(croom.courses().teachers(), 'list', 'teachers', page_message=page_message, courseId=entity) for teacher in teachers: email = teacher['profile'].get('emailAddress', None) if email: users.append(email) if entity_type in ['courseparticipants', 'students']: page_message = 'Got %%total_items%% Students...' - students = callGAPIpages(croom.courses().students(), 'list', 'students', page_message=page_message, courseId=entity) + students = gapi.get_all_pages(croom.courses().students(), 'list', 'students', page_message=page_message, courseId=entity) for student in students: email = student['profile'].get('emailAddress', None) if email: @@ -13046,9 +12945,9 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No if not silent: printGettingAllItems('Users', None) page_message = 'Got %%total_items%% Users...' - all_users = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, - customer=GC_Values[GC_CUSTOMER_ID], query=query, - fields='nextPageToken,users(primaryEmail)') + all_users = gapi.get_all_pages(cd.users(), 'list', 'users', page_message=page_message, + customer=GC_Values[GC_CUSTOMER_ID], query=query, + fields='nextPageToken,users(primaryEmail)') for member in all_users: users.append(member['primaryEmail']) if not silent: @@ -13057,8 +12956,8 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No if not silent: printGettingAllItems('CrOS Devices', None) page_message = 'Got %%total_items%% CrOS Devices...' - all_cros = callGAPIpages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, - customerId=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,chromeosdevices(deviceId)') + all_cros = gapi.get_all_pages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, + customerId=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,chromeosdevices(deviceId)') for member in all_cros: users.append(member['deviceId']) if not silent: @@ -13081,9 +12980,9 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No if not silent: printGettingAllItems('CrOS Devices', query) page_message = 'Got %%total_items%% CrOS Devices...' - members = callGAPIpages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, - customerId=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,chromeosdevices(deviceId)', - query=query) + members = gapi.get_all_pages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, + customerId=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,chromeosdevices(deviceId)', + query=query) for member in members: deviceId = member['deviceId'] if deviceId not in usersSet: diff --git a/src/gapi/__init__.py b/src/gapi/__init__.py index 785cf689c..a2d147cb8 100644 --- a/src/gapi/__init__.py +++ b/src/gapi/__init__.py @@ -1,5 +1,7 @@ """Methods related to execution of GAPI requests.""" +import sys + import controlflow import display from gapi import errors @@ -8,8 +10,8 @@ from var import (GC_CA_FILE, GC_Values, GC_TLS_MIN_VERSION, GC_TLS_MAX_VERSION, GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER, GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID, - MESSAGE_API_ACCESS_CONFIG, MESSAGE_API_ACCESS_DENIED, - MESSAGE_SERVICE_NOT_APPLICABLE) + MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG, + MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE) import google.auth.exceptions @@ -138,12 +140,12 @@ def call(service, controlflow.system_error_exit(4, str(e)) -def get_first_page(service, - function, - items='items', - throw_reasons=None, - retry_reasons=None, - **kwargs): +def get_items(service, + function, + items='items', + throw_reasons=None, + retry_reasons=None, + **kwargs): """Gets a single page of items from a Google service function that is paged. Args: @@ -173,6 +175,128 @@ def get_first_page(service, return [] +def _get_max_page_size_for_api_call(service, function, **kwargs): + """Gets the maximum number of results supported for a single API call. + + Args: + service: A Google service object for the desired API. + function: String, The name of the service method to check for max page size. + **kwargs: Additional params that will be passed to the request method. + + Returns: + Int, A value from discovery if it exists, otherwise value from + MAX_RESULTS_API_EXCEPTIONS, otherwise None + """ + method = getattr(service, function) + api_id = method(**kwargs).methodId + for resource in service._rootDesc.get('resources', {}).values(): + for a_method in resource.get('methods', {}).values(): + if a_method.get('id') == api_id: + if not a_method.get('parameters') or a_method['parameters'].get( + 'pageSize') or not a_method['parameters'].get('maxResults'): + # Make sure API call supports maxResults. For now we don't care to + # set pageSize since all known pageSize API calls have + # default pageSize == max pageSize. + return None + known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id) + max_results = a_method['parameters']['maxResults'].get( + 'maximum', known_api_max) + return {'maxResults': max_results} + + return None + + +def get_all_pages(service, + function, + items='items', + page_message=None, + message_attribute=None, + soft_errors=False, + throw_reasons=None, + retry_reasons=None, + **kwargs): + """Aggregates and returns all pages of a Google service function response. + + All pages of items are aggregated and returned as a single list. + + Args: + service: A Google service object for the desired API. + function: String, The name of a service request method to execute. + items: String, the name of the resulting "items" field within the method's + response object. The items in this field will be aggregated across all + pages and returned. + page_message: String, a message to be displayed to the user during paging. + Template strings allow for dynamic content to be inserted during paging. + Supported template strings: + %%total_items%% : The current number of items discovered across all + pages. + %%first_item%% : In conjunction with `message_attribute` arg, will + display a unique property of the first item in the current page. + %%last_item%% : In conjunction with `message_attribute` arg, will + display a unique property of the last item in the current page. + message_attribute: String, the name of a signature field within a single + returned item which identifies that unique item. This field is used with + `page_message` to templatize a paging status message. + soft_errors: Bool, If True, writes non-fatal errors to stderr. + throw_reasons: A list of Google HTTP error reason strings indicating the + errors generated by this request should be re-thrown. All other HTTP + errors are consumed. + retry_reasons: A list of Google HTTP error reason strings indicating which + error should be retried, using exponential backoff techniques, when the + error reason is encountered. + **kwargs: Additional params to pass to the request method. + + Returns: + A list of all items received from all paged responses. + """ + if 'maxResults' not in kwargs and 'pageSize' not in kwargs: + page_key = _get_max_page_size_for_api_call(service, function, **kwargs) + if page_key: + kwargs.update(page_key) + all_items = [] + page_token = None + total_items = 0 + while True: + page = call( + service, + function, + soft_errors=soft_errors, + throw_reasons=throw_reasons, + retry_reasons=retry_reasons, + pageToken=page_token, + **kwargs) + if page: + page_token = page.get('nextPageToken') + page_items = page.get(items, []) + num_page_items = len(page_items) + total_items += num_page_items + all_items.extend(page_items) + else: + page_token = None + num_page_items = 0 + + # Show a paging message to the user that indicates paging progress + if page_message: + show_message = page_message.replace('%%total_items%%', str(total_items)) + if message_attribute: + first_item = page_items[0] if num_page_items > 0 else {} + last_item = page_items[-1] if num_page_items > 1 else first_item + show_message = show_message.replace( + '%%first_item%%', str(first_item.get(message_attribute, ''))) + show_message = show_message.replace( + '%%last_item%%', str(last_item.get(message_attribute, ''))) + sys.stderr.write('\r') + sys.stderr.flush() + sys.stderr.write(show_message) + + if not page_token: + # End the paging status message and return all items. + if page_message and (page_message[-1] != '\n'): + sys.stderr.write('\r\n') + sys.stderr.flush() + return all_items + + # TODO: Make this private once all execution related items that use this method # have been brought into this file def handle_oauth_token_error(e, soft_errors): diff --git a/src/gapi/__init___test.py b/src/gapi/__init___test.py index 60cbca867..679fa3959 100644 --- a/src/gapi/__init___test.py +++ b/src/gapi/__init___test.py @@ -79,6 +79,40 @@ def setUp(self): self.mock_service = MagicMock() self.mock_method_name = 'mock_method' self.mock_method = getattr(self.mock_service, self.mock_method_name) + + self.simple_3_page_response = [ + { + 'items': [{ + 'position': 'page1,item1' + }, { + 'position': 'page1,item2' + }, { + 'position': 'page1,item3' + }], + 'nextPageToken': 'page2' + }, + { + 'items': [{ + 'position': 'page2,item1' + }, { + 'position': 'page2,item2' + }, { + 'position': 'page2,item3' + }], + 'nextPageToken': 'page3' + }, + { + 'items': [{ + 'position': 'page3,item1' + }, { + 'position': 'page3,item2' + }, { + 'position': 'page3,item3' + }], + }, + ] + self.empty_items_response = {'items': []} + super(GapiTest, self).setUp() def test_call_returns_basic_200_response(self): @@ -236,38 +270,269 @@ def test_call_retries_requests_with_backoff_on_servernotfounderror( # Make sure a backoff technique was used for retry. self.assertEqual(mock_wait_on_failure.call_count, 1) - def test_get_first_page_calls_correct_service_function(self): - pass + def test_get_items_calls_correct_service_function(self): + gapi.get_items(self.mock_service, self.mock_method_name) + self.assertTrue(self.mock_method.called) - def test_get_first_page_returns_one_page(self): + def test_get_items_returns_one_page(self): fake_response = {'items': [{}, {}, {}]} self.mock_method.return_value.execute.return_value = fake_response - page = gapi.get_first_page(self.mock_service, self.mock_method_name) + page = gapi.get_items(self.mock_service, self.mock_method_name) self.assertEqual(page, fake_response['items']) - def test_get_first_page_non_standard_page_field_name(self): + def test_get_items_non_default_page_field_name(self): field_name = 'things' fake_response = {field_name: [{}, {}, {}]} self.mock_method.return_value.execute.return_value = fake_response - page = gapi.get_first_page( + page = gapi.get_items( self.mock_service, self.mock_method_name, items=field_name) self.assertEqual(page, fake_response[field_name]) - def test_get_first_page_passes_additional_kwargs_to_service(self): - gapi.get_first_page( + def test_get_items_passes_additional_kwargs_to_service(self): + gapi.get_items( self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2) - self.mock_method.assert_called_once() + self.assertEqual(self.mock_method.call_count, 1) method_kwargs = self.mock_method.call_args[1] self.assertEqual(1, method_kwargs.get('my_param_1')) self.assertEqual(2, method_kwargs.get('my_param_2')) - def test_get_first_page_returns_empty_list_when_no_items_returned(self): + def test_get_items_returns_empty_list_when_no_items_returned(self): non_items_response = {'noItemsInThisResponse': {}} self.mock_method.return_value.execute.return_value = non_items_response - page = gapi.get_first_page(self.mock_service, self.mock_method_name) + page = gapi.get_items(self.mock_service, self.mock_method_name) self.assertIsInstance(page, list) self.assertEqual(0, len(page)) + def test_get_all_pages_returns_all_items(self): + page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': '2'} + page_2 = {'items': ['2-1', '2-2', '2-3'], 'nextPageToken': '3'} + page_3 = {'items': ['3-1', '3-2', '3-3']} + self.mock_method.return_value.execute.side_effect = [page_1, page_2, page_3] + response_items = gapi.get_all_pages(self.mock_service, + self.mock_method_name) + self.assertListEqual(response_items, + page_1['items'] + page_2['items'] + page_3['items']) + + def test_get_all_pages_includes_next_pagetoken_in_request(self): + page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': 'someToken'} + page_2 = {'items': ['2-1', '2-2', '2-3']} + self.mock_method.return_value.execute.side_effect = [page_1, page_2] + + gapi.get_all_pages(self.mock_service, self.mock_method_name, pageSize=100) + self.assertEqual(self.mock_method.call_count, 2) + call_2_kwargs = self.mock_method.call_args_list[1][1] + self.assertIn('pageToken', call_2_kwargs) + self.assertEqual(call_2_kwargs['pageToken'], page_1['nextPageToken']) + + def test_get_all_pages_uses_default_max_page_size(self): + sample_api_id = list(gapi.MAX_RESULTS_API_EXCEPTIONS.keys())[0] + sample_api_max_results = gapi.MAX_RESULTS_API_EXCEPTIONS[sample_api_id] + self.mock_method.return_value.methodId = sample_api_id + self.mock_service._rootDesc = { + 'resources': { + 'someResource': { + 'methods': { + 'someMethod': { + 'id': sample_api_id, + 'parameters': { + 'maxResults': { + 'maximum': sample_api_max_results + } + } + } + } + } + } + } + self.mock_method.return_value.execute.return_value = self.empty_items_response + + gapi.get_all_pages(self.mock_service, self.mock_method_name) + request_method_kwargs = self.mock_method.call_args[1] + self.assertIn('maxResults', request_method_kwargs) + self.assertEqual(request_method_kwargs['maxResults'], + gapi.MAX_RESULTS_API_EXCEPTIONS.get(sample_api_id)) + + def test_get_all_pages_max_page_size_overrided(self): + self.mock_method.return_value.execute.return_value = self.empty_items_response + + gapi.get_all_pages( + self.mock_service, self.mock_method_name, pageSize=123456) + request_method_kwargs = self.mock_method.call_args[1] + self.assertIn('pageSize', request_method_kwargs) + self.assertEqual(123456, request_method_kwargs['pageSize']) + + @patch.object(gapi.sys.stderr, 'write') + def test_get_all_pages_prints_paging_message(self, mock_write): + self.mock_method.return_value.execute.side_effect = self.simple_3_page_response + + paging_message = 'A simple string displayed during paging' + gapi.get_all_pages( + self.mock_service, self.mock_method_name, page_message=paging_message) + messages_written = [ + call_args[0][0] for call_args in mock_write.call_args_list + ] + self.assertIn(paging_message, messages_written) + + @patch.object(gapi.sys.stderr, 'write') + def test_get_all_pages_prints_paging_message_inline(self, mock_write): + self.mock_method.return_value.execute.side_effect = self.simple_3_page_response + + paging_message = 'A simple string displayed during paging' + gapi.get_all_pages( + self.mock_service, self.mock_method_name, page_message=paging_message) + messages_written = [ + call_args[0][0] for call_args in mock_write.call_args_list + ] + + # Make sure a return carriage was written between two pages + paging_message_call_positions = [ + i for i, message in enumerate(messages_written) + if message == paging_message + ] + self.assertGreater(len(paging_message_call_positions), 1) + printed_between_page_messages = messages_written[ + paging_message_call_positions[0]:paging_message_call_positions[1]] + self.assertIn('\r', printed_between_page_messages) + + @patch.object(gapi.sys.stderr, 'write') + def test_get_all_pages_ends_paging_message_with_newline(self, mock_write): + self.mock_method.return_value.execute.side_effect = self.simple_3_page_response + + paging_message = 'A simple string displayed during paging' + gapi.get_all_pages( + self.mock_service, self.mock_method_name, page_message=paging_message) + messages_written = [ + call_args[0][0] for call_args in mock_write.call_args_list + ] + last_page_message_index = len( + messages_written) - messages_written[::-1].index(paging_message) + last_carriage_return_index = len( + messages_written) - messages_written[::-1].index('\r\n') + self.assertGreater(last_carriage_return_index, last_page_message_index) + + @patch.object(gapi.sys.stderr, 'write') + def test_get_all_pages_prints_attribute_total_items_in_paging_message( + self, mock_write): + self.mock_method.return_value.execute.side_effect = self.simple_3_page_response + + paging_message = 'Total number of items discovered: %%total_items%%' + gapi.get_all_pages( + self.mock_service, self.mock_method_name, page_message=paging_message) + + messages_written = [ + call_args[0][0] for call_args in mock_write.call_args_list + ] + page_1_item_count = len(self.simple_3_page_response[0]['items']) + page_1_message = paging_message.replace('%%total_items%%', + str(page_1_item_count)) + self.assertIn(page_1_message, messages_written) + + page_2_item_count = len(self.simple_3_page_response[1]['items']) + page_2_message = paging_message.replace( + '%%total_items%%', str(page_1_item_count + page_2_item_count)) + self.assertIn(page_2_message, messages_written) + + page_3_item_count = len(self.simple_3_page_response[2]['items']) + page_3_message = paging_message.replace( + '%%total_items%%', + str(page_1_item_count + page_2_item_count + page_3_item_count)) + self.assertIn(page_3_message, messages_written) + + # Assert that the template text is always replaced. + for message in messages_written: + self.assertNotIn('%%total_items', message) + + @patch.object(gapi.sys.stderr, 'write') + def test_get_all_pages_prints_attribute_first_item_in_paging_message( + self, mock_write): + self.mock_method.return_value.execute.side_effect = self.simple_3_page_response + + paging_message = 'First item in page: %%first_item%%' + gapi.get_all_pages( + self.mock_service, + self.mock_method_name, + page_message=paging_message, + message_attribute='position') + + messages_written = [ + call_args[0][0] for call_args in mock_write.call_args_list + ] + page_1_message = paging_message.replace( + '%%first_item%%', + self.simple_3_page_response[0]['items'][0]['position']) + self.assertIn(page_1_message, messages_written) + + page_2_message = paging_message.replace( + '%%first_item%%', + self.simple_3_page_response[1]['items'][0]['position']) + self.assertIn(page_2_message, messages_written) + + # Assert that the template text is always replaced. + for message in messages_written: + self.assertNotIn('%%first_item', message) + + @patch.object(gapi.sys.stderr, 'write') + def test_get_all_pages_prints_attribute_last_item_in_paging_message( + self, mock_write): + self.mock_method.return_value.execute.side_effect = self.simple_3_page_response + + paging_message = 'Last item in page: %%last_item%%' + gapi.get_all_pages( + self.mock_service, + self.mock_method_name, + page_message=paging_message, + message_attribute='position') + + messages_written = [ + call_args[0][0] for call_args in mock_write.call_args_list + ] + page_1_message = paging_message.replace( + '%%last_item%%', + self.simple_3_page_response[0]['items'][-1]['position']) + self.assertIn(page_1_message, messages_written) + + page_2_message = paging_message.replace( + '%%last_item%%', + self.simple_3_page_response[1]['items'][-1]['position']) + self.assertIn(page_2_message, messages_written) + + # Assert that the template text is always replaced. + for message in messages_written: + self.assertNotIn('%%last_item', message) + + def test_get_all_pages_prints_all_attributes_in_paging_message(self): + pass + + def test_get_all_pages_passes_additional_kwargs_to_service_method(self): + self.mock_method.return_value.execute.return_value = self.empty_items_response + gapi.get_all_pages( + self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2) + method_kwargs = self.mock_method.call_args[1] + self.assertEqual(method_kwargs.get('my_param_1'), 1) + self.assertEqual(method_kwargs.get('my_param_2'), 2) + + @patch.object(gapi, 'call') + def test_get_all_pages_passes_throw_and_retry_reasons(self, mock_call): + throw_for = MagicMock() + retry_for = MagicMock() + mock_call.return_value = self.empty_items_response + gapi.get_all_pages( + self.mock_service, + self.mock_method_name, + throw_reasons=throw_for, + retry_reasons=retry_for) + method_kwargs = mock_call.call_args[1] + self.assertEqual(method_kwargs.get('throw_reasons'), throw_for) + self.assertEqual(method_kwargs.get('retry_reasons'), retry_for) + + def test_get_all_pages_non_default_items_field_name(self): + field_name = 'things' + fake_response = {field_name: [{}, {}, {}]} + self.mock_method.return_value.execute.return_value = fake_response + page = gapi.get_all_pages( + self.mock_service, self.mock_method_name, items=field_name) + self.assertEqual(page, fake_response[field_name]) + if __name__ == '__main__': unittest.main()