Improvements to Productivity Tracker

python/productivity_tracker/.gcloudignore

@@ -6,7 +6,10 @@
 # For more information, run:
 #   $ gcloud topic gcloudignore
 #
-.gcloudignore
+
+# Ignore everything git ignores
+#!include:.gitignore
+
 # If you would like to upload your .git directory, .gitignore file or files
 # from your .gitignore file, remove the corresponding line
 # below:

python/productivity_tracker/.gitignore

@@ -20,3 +20,6 @@ venv/
 
 # sql proxy
 cloud_sql_proxy
+
+# secret settings
+settings.txt

python/productivity_tracker/app.yaml

@@ -2,5 +2,4 @@ runtime: python37
 entrypoint: gunicorn -b :$PORT productivity_tracker.wsgi
 
 env_variables:
-  ENABLE_NLP: True
-
+  ENABLE_NLP: True

python/productivity_tracker/productivity_bot/api_helper.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import google.auth
 from os import environ
 from googleapiclient.discovery import build
 from google.oauth2 import service_account
@@ -25,8 +26,8 @@ class APIHelper:
         'https://www.googleapis.com/auth/drive.file'
     ]
 
-    def __init__(self, keyfile_name):
-        credentials = service_account.Credentials.from_service_account_file(keyfile_name)
+    def __init__(self):
+        credentials, _ = google.auth.default()
         self.credentials = credentials.with_scopes(scopes=APIHelper.scopes)
         self.sheets_service = build('sheets', 'v4', credentials=self.credentials)
         self.drive_service = build('drive', 'v3', credentials=self.credentials)
@@ -185,15 +186,15 @@ def add_nlp_columns(data):
             people = APIHelper.filter_nlp_results(
                 entities,
                 nlp.ENTITY_TYPE,
-                lambda token: token.type,
+                lambda token: token.type_,
                 lambda token: token.name,
                 lambda tag: tag == 'PERSON',
             )
             # Extract all non-persons from entities list
             other_parties = APIHelper.filter_nlp_results(
                 entities,
                 nlp.ENTITY_TYPE,
-                lambda token: token.type,
+                lambda token: token.type_,
                 lambda token: token.name,
                 lambda tag: tag != 'PERSON',
             )

python/productivity_tracker/productivity_bot/apps.py

@@ -16,4 +16,5 @@
 
 
 class ProductivityBotConfig(AppConfig):
+    default_auto_field = 'django.db.models.AutoField'
     name = 'productivity_bot'

python/productivity_tracker/productivity_bot/inbound_message_handlers.py

@@ -90,7 +90,7 @@ def end_active_loop(user):
     if not active_loop:
         return "You are not in a working session."
 
-    api_helper = APIHelper('service_account.json')
+    api_helper = APIHelper()
 
     sheet_properties, file_existed = api_helper.get_or_create_sheet(user)
     sheet_id = sheet_properties['sheetId']

python/productivity_tracker/productivity_bot/message_users_helpers.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import google.auth
 from googleapiclient.discovery import build
 from google.oauth2 import service_account
 
@@ -25,7 +26,7 @@ def send_reminder(space_name):
     response = {'text': 'What have you completed since I last checked in?'}
 
     scopes = ['https://www.googleapis.com/auth/chat.bot']
-    credentials = service_account.Credentials.from_service_account_file('service_account.json')
+    credentials, _ = google.auth.default()
     credentials = credentials.with_scopes(scopes=scopes)
     chat = build('chat', 'v1', credentials=credentials)
     chat.spaces().messages().create(

python/productivity_tracker/productivity_bot/models.py

@@ -28,6 +28,8 @@ class User(models.Model):
     email = models.CharField(max_length=100)
     spreadsheet_id = models.CharField(max_length=100, blank=True, default="")
 
+    def __str__(self):
+        return str(self.email)
 
 class ActiveLoops(models.Model):
     ''' To store all of the users that are currently in a working session
@@ -41,6 +43,13 @@ class ActiveLoops(models.Model):
         related_name="active_loop"
     )
 
+    def __str__(self):
+        return f"Activity loop for {self.user}"
+
+    class Meta:
+        verbose_name = "Activity Loop"
+
+
 
 class UserResponses(models.Model):
     ''' To store user responses during active loops
@@ -55,3 +64,9 @@ class UserResponses(models.Model):
         related_name="user_responses"
     )
     raw_text = models.CharField(max_length=300)
+
+    def __str__(self):
+        return f"User response in {self.active_loop}"
+
+    class Meta:
+        verbose_name = "User Response"

python/productivity_tracker/productivity_bot/nlp_helper.py

@@ -13,14 +13,14 @@
 # limitations under the License.
 
 from google.cloud import language
-from google.cloud.language import enums
-from google.cloud.language import types
+
 
 class NLPHelper:
     ENTITY_TYPE = ('UNKNOWN', 'PERSON', 'LOCATION', 'ORGANIZATION',
                    'EVENT', 'WORK_OF_ART', 'CONSUMER_GOOD', 'OTHER')
     POS_TAG = ('UNKNOWN', 'ADJ', 'ADP', 'ADV', 'CONJ', 'DET', 'NOUN', 'NUM',
                'PRON', 'PRT', 'PUNCT', 'VERB', 'X', 'AFFIX')
+
     def __init__(self):
         self.client = language.LanguageServiceClient()
 
@@ -41,10 +41,12 @@ def analyze_text(self, text, analysis_type):
                         (see here:
                         https://cloud.google.com/natural-language/docs/reference/rest/v1/Entity)
         """
-        document = types.Document(
-            content=text,
-            type=enums.Document.Type.PLAIN_TEXT)
+        document = {"content": text, "type_": language.Document.Type.PLAIN_TEXT}
 
-        if analysis_type == 'entities':
-            return self.client.analyze_entities(document).entities
-        return self.client.analyze_syntax(document).tokens
+        if analysis_type == "entities":
+            return self.client.analyze_entities(
+                document=document, encoding_type=language.EncodingType.UTF8
+            ).entities
+        return self.client.analyze_syntax(
+            document=document, encoding_type=language.EncodingType.UTF8
+        ).tokens

python/productivity_tracker/productivity_tracker/settings.py

@@ -12,7 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import io
 import os
+from google.cloud import secretmanager
+import environ
+
+env = environ.Env(DEBUG=(bool, False))
+
+if os.environ.get("GOOGLE_CLOUD_PROJECT", None):
+    client = secretmanager.SecretManagerServiceClient()
+    name = f"projects/{os.environ.get('GOOGLE_CLOUD_PROJECT')}/secrets/django_settings/versions/latest"
+    payload = client.access_secret_version(name=name).payload.data.decode("UTF-8")
+    env.read_env(io.StringIO(payload))
+else:
+    raise Exception("No GOOGLE_CLOUD_PROJECT detected. No secrets found.")
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -22,7 +35,7 @@
 # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = 'w8m4g!u-(%3&x@bxw4ohekaa-us4a0#xvwe%br52zpgw1@j*66'
+SECRET_KEY = env("SECRET_KEY")
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
@@ -87,40 +100,29 @@
     pymysql.install_as_MySQLdb()
 
 # [START db_setup]
-if os.getenv('GAE_INSTANCE'):
-    # Running on production App Engine, so connect to Google Cloud SQL using
-    # the unix socket at /cloudsql/<your-cloudsql-connection string>
-    DATABASES = {
-        'default': {
-            # If you are using Cloud SQL for PostgreSQL rather than MySQL, set
-            # 'ENGINE': 'django.db.backends.postgresql' instead of the following.
-            'ENGINE': 'django.db.backends.mysql',
-            'HOST': '/cloudsql/<your-cloudsql-connection-string>',
-            'NAME': '<your-database-name>',
-            'USER': '<your-database-user>',
-            'PASSWORD': '<your-database-password>',
-            # For PostgresQL, set 'PORT': '5432' instead of the following. Any Cloud
-            # SQL Proxy instances running locally must also be set to tcp:3306.
-            'PORT': '3306',
-        }
+DATABASES = {
+    'default': {
+        # If you are using Cloud SQL for PostgreSQL rather than MySQL, set
+        # 'ENGINE': 'django.db.backends.postgresql' instead of the following.
+        'ENGINE': 'django.db.backends.mysql',
+        'HOST': f'/cloudsql/{env("DB_HOST")}',
+        'NAME': env("DB_NAME"),
+        'USER': env("DB_USER"),
+        'PASSWORD': env("DB_PASSWORD"),
+        # For PostgresQL, set 'PORT': '5432' instead of the following. Any Cloud
+        # SQL Proxy instances running locally must also be set to tcp:3306.
+        'PORT': '3306',
     }
-else:
+}
+
+if os.getenv('USE_CLOUD_SQL_AUTH_PROXY'):
     # Running locally so connect to either a local MySQL instance or connect to
     # Cloud SQL via the proxy. To start the proxy via command line:
     #
     #     $ cloud_sql_proxy -instances=[INSTANCE_CONNECTION_NAME]=tcp:3306
     #
     # See https://cloud.google.com/sql/docs/mysql-connect-proxy
-    DATABASES = {
-        'default': {
-            'ENGINE': 'django.db.backends.mysql',
-            'HOST': '127.0.0.1',
-            'PORT': '3306',
-            'NAME': '<your-database-name>',
-            'USER': '<your-database-user>',
-            'PASSWORD': '<your-database-password>',
-        }
-    }
+    DATABASES["default"]["HOST"] = "127.0.0.1"
 
 # Password validation
 # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
@@ -157,5 +159,6 @@
 
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/1.11/howto/static-files/
-
-STATIC_URL = '/static/'
+STATIC_ROOT = "static"
+STATIC_URL = "/static/"
+STATICFILES_DIRS = []

python/productivity_tracker/productivity_tracker/urls.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from django.conf import settings
 from django.conf.urls import url
+from django.conf.urls.static import static
 from django.contrib import admin
 from productivity_bot.views import ChatbotEvent, MessageUsers
 
 urlpatterns = [
     url(r'^admin/', admin.site.urls),
     url(r'^message_users', MessageUsers.as_view()),
     url(r'^chatbot_event', ChatbotEvent.as_view()),
-]
+] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

python/productivity_tracker/readme.md

@@ -23,15 +23,15 @@ This bot uses the following Google APIs:
 ### Commands
 
 - `start`:  Start a working session.
-- *example:  `submitted time reports` *:  During a working session, say anything to log work.
+- *example*:  `submitted time reports`:  During a working session, say anything to log work.
 - `stop`:  End a working session and get a report.
 
 # How to Install and Deploy to [Google App Engine Flex](https://cloud.google.com/appengine/)
 
 ### Requirements
 
 - A [Google Cloud Platform](https://cloud.google.com/) account where you can create a project and enable billing.
-- A local terminal with Python 2.7 and pip installed.
+- A local terminal with Python 3.7+ and pip installed.
 
 > Go [here](https://www.python.org/downloads/) to install the latest version of Python. Pip comes installed with most Python distributions.
 
@@ -50,25 +50,24 @@ This bot uses the following Google APIs:
     `source env/bin/activate`
 1. Install project dependencies:
     `pip install -r requirements.txt`
+    Note: you may need to `sudo apt-get install libmysqlclient-dev -y`
 
 ### 2. Set up on Cloud Console
 
 1. [Create a project on the Google Cloud Platform Console](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project)
 1. [Enable Billing on that Project](https://cloud.google.com/billing/docs/how-to/modify-project)
-1. Enable the following APIs in [Google Cloud's API Library](https://console.cloud.google.com/apis/library)
-    - [Cloud SQL API](https://cloud.google.com/sql/)
-    - [Google Drive API](https://developers.google.com/drive/)
-    - [Google Sheets API](https://developers.google.com/sheets/)
-    - [Cloud Natural Language API](https://cloud.google.com/natural-language/)
-    - [Hangouts Chat API](https://developers.google.com/hangouts/chat/)
+1. [Enable the Cloud SQL, Compute Engine, Google Drive, Google Sheets, Secret Manager, Cloud Natural Language and Hangouts Chat APIs](https://console.cloud.google.com/flows/enableapi?apiid=compute.googleapis.com,drive.googleapis.com,sqladmin.googleapis.com,sheets.googleapis.com,secretmanager.googleapis.com,language.googleapis.com,chat.googleapis.com).
 1. [Create a service account and download the service account key](https://cloud.google.com/iam/docs/creating-managing-service-accounts#creating_a_service_account).
     -  When creating the service account, select **Project** > **Owner** under **Project Role**
-    - Save the service account key as `service_account.json`.
 1. [Create a Cloud SQL MySQL instance](https://cloud.google.com/sql/docs/mysql/create-instance).
-1. Update `app.yaml` to set the [connection instance name](https://cloud.google.com/sql/docs/mysql/connect-app-engine#configuring).
-1. Update `settings.py` to set the database connection settings.
+1. Update `settings.txt` to set the databse connection settings and secret keys.
+1. Secure the settings file in Secret Manager: 
+    `gcloud secrets create django_settings --data-file settings.txt`
 
 ### 3. Deployment
 
-1. [Deploy the application](https://cloud.google.com/python/django/flexible-environment).
-1. [Configure the bot](https://developers.google.com/hangouts/chat/how-tos/bots-publish). Set "https://{PROJECT_NAME}.appspot.com/chatbot_event" as the bot url.
+1. Run the application on your local machine and [run the database and static migration commands](https://cloud.google.com/python/django/app-engine#run-locally). 
+    Note: you will have to use `tcp=3306` for MySQL
+1. Deploy the application with the service account: 
+    `gcloud beta app deploy --service-account SERVICEACCOUNT@PROJECT_ID.iam.gserviceaccount.com`
+1. [Configure the bot](https://developers.google.com/hangouts/chat/how-tos/bots-publish) in the [console](https://console.cloud.google.com/apis/api/chat.googleapis.com/hangouts-chat). Set "https://{PROJECT_NAME}.appspot.com/chatbot_event" as the bot url.

python/productivity_tracker/requirements.txt

@@ -1,8 +1,10 @@
-Django==2.2.13
-pytz==2019.3
-jsonpickle==1.2
-google-cloud-language==1.3.0
-google-api-python-client==1.7.11
-google-auth==1.6.3
-gunicorn==19.9.0
-mysqlclient==1.4.5
+Django==3.2.6
+pytz==2021.1
+jsonpickle==2.0.0
+google-cloud-language==2.2.2
+google-api-python-client==2.15.0
+google-auth==1.34.0
+gunicorn==20.1.0
+pymysql==1.0.2
+django-environ==0.4.5
+google-cloud-secret-manager==2.7.0

python/productivity_tracker/settings.txt

@@ -0,0 +1,5 @@
+DB_HOST=<PROJECT>:<REGION>:<CLOUDSQL_INSTANCE>
+DB_NAME=<DATABASE_NAME>
+DB_USER=<DATABASE_USER>
+DB_PASSWORD=<DATABASE_PASSWORD>
+SECRET_KEY=<random-string-here>