diff --git a/tm_admin/tasks/tasks.py b/tm_admin/tasks/tasks.py index 504cbe5..f3ea6d3 100755 --- a/tm_admin/tasks/tasks.py +++ b/tm_admin/tasks/tasks.py @@ -181,11 +181,16 @@ async def mergeHistory(self, entry = {"user_id": record['user_id']} # entry['action'] = Taskaction(record['action']).name entry['action'] = record['action'] - entry['action_text'] = record['action_text'] + # The action_text column often has issues with embedded + # quotes. + if record['action_text']: + fix = record['action_text'].replace('"', '"') + entry['action_text'] = fix.replace("'", ''').replace("\xa0", "") if record['action_date']: entry['action_date'] = '{:%Y-%m-%dT%H:%M:%S}'.format(record['action_date']) # If there is an issue, add it to the record in the jsonb column if record['id'] in issues: + entry.update(issues[record['id']]) entry.update(issues[record['id']]) # entry['issue'] = issues['issue'] # entry['category'] = issues['category'] @@ -195,6 +200,12 @@ async def mergeHistory(self, # print(sql) queries.append(sql) + # Add a timestamp to the created column so this table can + # be partitioned. + sql = f"UPDATE tasks SET created = '{entry['action_date']}' WHERE id={record['task_id']} AND project_id={record['project_id']}" + # print(sql) + queries.append(sql) + entries = len(queries) chunk = round(entries/cores) import copy @@ -204,6 +215,8 @@ async def mergeHistory(self, outpg = PostgresClient() # FIXME: this should not be hard coded await outpg.connect('localhost/tm_admin') + # FIXME: this may be removed after testing, but memory + # corruption errors fored this workaround. foo = copy.copy(queries[block:block + chunk -1]) log.debug(f"Dispatching thread {block}:{block + chunk - 1}") # await updateThread(foo, outpg) diff --git a/tm_admin/tasks/tasks.yaml b/tm_admin/tasks/tasks.yaml index e33b26b..a99fc61 100644 --- a/tm_admin/tasks/tasks.yaml +++ b/tm_admin/tasks/tasks.yaml @@ -12,14 +12,16 @@ - int32 - required: true - unique: true + - created: + - timestamp + - required: true + - share: false + - partition: true - project_task_index: - int32 - project_task_name: - string - share: true -# - outline: -# - polygon -# - share: true - geometry: - polygon - share: true @@ -47,5 +49,5 @@ - int32 - history: - jsonb - - invalidation_history: + - issues: - jsonb