From fa006c3f6da1ca696db291dd192114b8ad2818ea Mon Sep 17 00:00:00 2001 From: hemanta212 Date: Fri, 18 Jun 2021 09:14:25 +0000 Subject: [PATCH] Rewrite of docs --- .gitignore | 24 ++--- README.md | 120 ++++++++++++++------- docs/manual_install.md | 233 ---------------------------------------- docs/postgres_setup.md | 51 +++++++++ docs/venv.md | 35 ++++++ flask_final/__init__.py | 2 +- flask_final/config.py | 67 +++++------- manage.py | 27 ++--- run.py | 36 +++---- template_secrets.json | 4 +- 10 files changed, 243 insertions(+), 356 deletions(-) delete mode 100644 docs/manual_install.md create mode 100644 docs/postgres_setup.md create mode 100644 docs/venv.md diff --git a/.gitignore b/.gitignore index 505bc3d..d7d7de8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,13 @@ -*venv* -*vscode* -*pyc -*pycache* -*vscode -*vim* -*swp -*swo -*swn -*.lock* -*.orig* -*poetry* +__pycache__/ +*venv/ +/.vscode/ +*.pyc +*.vim +*.swp +*.swo +*.swn +*.lock +*.orig secrets* -*.db* +*.db *.env diff --git a/README.md b/README.md index 7a58bea..0fdcc1d 100644 --- a/README.md +++ b/README.md @@ -12,63 +12,107 @@ Features: * News collection(scraping, api) and serving * Pagination, Collapsible sidebar, Responsive navigation * Use proper structure with flask blueprinting -* Support PostgreSQL and Sqlite database * Hosted Live at heroku @ kbd.herokuapp.com -Todo: -* Tick indcates progress. +Todos: + - [x] Integrate various international news apis(top, general, sports, tech...) -- [ ] News search on keyword for custom category (like bitcoin, trump, etc) -- [ ] Give user choice to customize news topic, sources, language etc -- [ ] Remove compulsory login and establish per person db record of preferences +- [x] Rewrite manage.py and database management +- [ ] Add logging +- [ ] Add tests +- [ ] Experiment with docker deploys +- [ ] Extract the scraper to separate news API - [ ] Redesign the dashboard +- [ ] Remove compulsory login and establish per person db record of preferences +- [ ] Give user choice to customize news topic, sources, language etc +- [ ] News search on keyword for custom category (like bitcoin, trump, etc) - [ ] Integrate social media login (facebook, github) -- [ ] Use Docker -- [x] Add logging -- [x] Add tests -- [ ] Rewrite manage.py and database management -## Installation -### Fast-Track test installation -* First properly install python 3.6 or above in your system. -* clone/download this repository and navigate to this repo through cmd -* Run +## Installation and Usage - python setupenv.py sqlite +- Install [python 3.6 or above](https://python.org/downloads) +- Clone/download this repository and navigate to this repo through cmd - OR +```sh +$ git clone https://github.com/hemanta212/nepali-news-portal-kbd +``` - python setupenv.py postgres +- Install either [poetry](https://github.com/python-poetry/poetry) or follow [this guide](/docs/venv.md) for setting up virtual environment. +- Installing dependencies using poetry +```sh +$ poetry install +``` +- Installing dependencies using pip. +```sh +$ python -m pip install -r requirements.txt +``` + +### Setting up Database +You can setup any database supported [here](https://docs.sqlalchemy.org/en/13/core/engines.html#supported-databases). This doc covers setting up sqlite and postgres db. + +##### Setting up SQLite +Populate the following as your environment variables + +``` +SECRET_KEY=" +Gunicorn is a production WSGI server that is essential for running flask project in production environments like with Procfile in [Heroku](https://heroku.com). +After installing, you just provide the application instance of project to gunicorn. + +eg. +``` +gunicorn run:app // prod env + +gunicorn run:app debug +``` + +#### *NOTE*: +Gunicorn is not supported in windows operating system. However, you can run it using [WSL](https://google.com/search?query=windows%20subsytem%20for%20linux). + +## Accessing all news +After successfully running the app, go to localhost:5000/signup and sign up with an account with this admin provisioned email 'try@try.com' (yes this exact email only) and login. You should see news from all the sources. diff --git a/docs/manual_install.md b/docs/manual_install.md deleted file mode 100644 index 6a1c64e..0000000 --- a/docs/manual_install.md +++ /dev/null @@ -1,233 +0,0 @@ -# Installation - -## Initial Steps -* First properly install python 3.6 or above in your system. -* clone/download this repository and navigate to this repo through cmd - -## Installation methods -You can go with either of ways of installation - -* [Poetry (Recommended)](#poetry) - 1. [Installing poetry](#set_1) - 2. [Installing packages and running the application](#run_1) - -* [Pip and venv (conventional way)](#pip) - 1. [Setting venv](#set_2) - 2. [Installing packages and running the application](#run_2) - -## Running with other databases -Running with other databases is simple, you setup any database you like that is supported with flask-sqlalchemy. [See here for supported database list.](https://docs.sqlalchemy.org/en/13/core/engines.html#supported-databases) Then you set its URI as a value to ```SQLALCHEMY_DATABASE_URI``` in secrets.json. - -You can set every variables present in secrets.json as your environment variables and get rid of secrets.json entirely. [See here for more info](#pg) - -* [Running with postgres database](#pg) - * [Windows setup](#windows_pg) - * [Unix setup](#unix_pg) - -## All ways of running the application -* [Running from manage.py file](#managepy) -* [Running from python and run.py file](#runpy) -* [Running from gunicorn (Not supported on windows)](#gunicorn) - - -### Poetry -[Poetry](https://github.com/python-poetry/poetry) helps you declare, manage and install dependencies of Python projects, ensuring you have the right stack everywhere. - -#### Installing poetry -* [Install poetry](https://github.com/python-poetry/poetry/#installation) - -``` -curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -``` -Restart the shell. - -#### Installing the packages and running project -* Rename 'template_secrets.json' to 'secrets.json'. -* Run, -``` -poetry install -poetry run python manage.py sqlite -poetry run python run.py secrets -``` - -* Finally open your browser and head over to http://localhost:5000 and website will load. - -##### *Notes*: -- Fill secrets.json file with your email and password for password reset functionality! -- Also, For enabling international news section, get an api from [News api](https://newsapi.org/register) and set environment variable named ```NEWS_API_KEY``` to that token value. - - -### Pip and venv -Both pip and venv are provided with the installation of python. -To make sure you have them just go to the cmd prompt and type and hit enter python -m pip and python -m venv respectively. - -#### Setting venv - -* Make a virtual environment: - -```python -m venv venv``` -* Activate it - - For Windows: - - ```venv\Scripts\activate``` - - - For Unix: - - ```source venv/bin/activate``` - -* Rename 'template_secrets.json' to 'secrets.json'. - -#### Installing packages and running the application -Here we have set the secrets file with necessary configs so we can use it to run. To know about other ways look [here](#runpy). - -``` -pip install -r requirements.txt -python manage.py sqlite -python run.py secrets -``` - -* Finally open your browser and head over to http://localhost:5000 and website will load. - - -##### *Notes*: -- Fill secrets.json file with your email and password for password reset functionality! -- Also, For enabling international news section, get an api from [News api](https://newsapi.org/register) and set environment variable named ```NEWS_API_KEY``` to that token value. - - -### Run using postgres database. -To run the project on posgres database you need to install postgresql 10+ in your system - -#### Windows setup: -Download and install [official site](https://www.postgresql.org/download/windows/) - -1. Create a postgreql database and obtain its local url -2. Now remove the gunicorn in requirements.txt. Then, - -```pip install -r requirements.txt``` - -3. Add your postgresql local url of database you created earlier to environment variable named ```DATABASE_URL``` -4. Make new SECRET_KEY environment variable and random string as value -5. *Note*: - - Add EMAIL and EMAIL_PASSWORD environment variable with your email and password for password reset functionality! - -6. Finally, -* If you are setting first time without migrate folder - -``` - python manage.py db init - python manage.py db migrate - python manage.py db upgrade - python manage.py runserver -``` -* If migrate folder is present - -``` - python manage.py db upgrade - python manage.py runserver -``` - -#### Linux setup: -1. Install postgres: - -```sudo apt-get install postgresql postgresql-contrib``` - -2. Now create a superuser for PostgreSQL - -```sudo -u postgres createuser --superuser name_of_user``` - -3. And create a database using created user account - -```sudo -u name_of_user createdb name_of_database``` - -4. You can access created database with created user by, - -```psql -U name_of_user -d name_of_database``` - -5. Your postgres database url wil be something like - -```postgresql://localhost/name_of_database``` - -6. Delete the secrets.json file if present in your folder. -7. Set environment variables named DATABASE_URL, EMAIL, and EMAIL_PASSWORD - -``` -export DATABASE_URL='postgresql://localhost/name_of_database' -export EMAIL='your_email@something.com' -export EMAIL_PASSWORD='your password' -``` -7. *Note*: - - EMAIL and EMAIL_PASSWORD are optional and only required for password reset functionality! - -8. Installation - * With Poetry ([see installation](#set_1)) - - ```poetry install``` - - * With Pip - - ```pip install -r requirements.txt``` - -9. Database Migration - * If migrations folder is present - - ```python manage.py db upgrade``` - - * Otherwise, - ``` - python manage.py db init - python manage.py db migrate - python manage.py db upgrade - ``` - -10. Running. - -```python manage.py runserver``` - -* Website will be at http://localhost:5000 load it in your browser. - - -## All ways of running the application -### Running from manage.py file -The 'manage.py' is the single file to manage your project. It will help you handle db migrations, db inits and run the project itself. It is the general convention in python web projects to have a 'manage.py' (or similar) file. - -The way I have set this up is runserver command will run the production version of app reading configs from environment variables. - -```python manage.py runserver``` - -This is production settings and if you need to run other configuration then ['run.py'](#runpy) is best method - -### Running from python and run.py file -'run.py' is a custom python file to run the project with diffrent configurations. -You specify the configuration type as arguments and then it will run the app. - -```python run.py ``` - -* Configuration types are. -- sqlite-debug -- sqlite-prod -- database-prod -- database-debug -- secrets (reads config from secrets.json file) - -eg. - -```python run.py sqlite-debug``` - -If no arguments is provided the default configuration is set as Database production which reads every config from environment variable - -This run.py script is used while deploying in platfroms like heroku. Where gunicorn calls this run.py script. See [gunicorn](#gunicorn) for more info. - -### Running from gunicorn (Not supported on windows) -Gunicorn is a production WSGI server that is essential for running flask project in production mode. - -After installing, you just provide the application instance of project to gunicorn. - -eg. -``` -gunicorn run:app #uses default config of run.py -gunicorn run:app sqlite-debug -gunicorn run:app database-debug -``` - -#### *NOTE*: -Gunicorn is not supported in windows operating system. You can run it on [WSL](https://google.com/search?query=windows%20subsytem%20for%20linux) however. diff --git a/docs/postgres_setup.md b/docs/postgres_setup.md new file mode 100644 index 0000000..05cee06 --- /dev/null +++ b/docs/postgres_setup.md @@ -0,0 +1,51 @@ +## Setup with postgres database + + * [Unix setup](#unix_pg) + * [Windows setup (WIP)](#windows_pg) + +### Run using postgres database. +To run the project on posgres database you need to install postgresql 10+ in your system + +#### Linux setup: +1. Install postgres: + +``` +sudo apt-get install postgresql postgresql-contrib +``` + +2. Now create a superuser for PostgreSQL + +``` +sudo -u postgres createuser --superuser name_of_user +``` + +3. And create a database using created user account + +``` +sudo -u name_of_user createdb name_of_database +``` + +4. You can access created database with created user by, + +``` +psql -U name_of_user -d name_of_database +``` + +5. Your postgres database url wil be something like + +``` +postgresql://localhost/name_of_database +``` +Or, if you have setup password then, + +``` +postgresql://user_name:password/localhost/name_of_database +``` + +7. Now take this url and go back to [this section in Main Readme file](../README.md#setting up-the-postgres-databse) + +``` +#### Windows setup: +Download and install [official site](https://www.postgresql.org/download/windows/) + +1. Create a postgreql database and obtain its local url diff --git a/docs/venv.md b/docs/venv.md new file mode 100644 index 0000000..39f08c4 --- /dev/null +++ b/docs/venv.md @@ -0,0 +1,35 @@ +### Setting up a Virtual Environment +Venv is provided with the default installation of python since python 3.4 and above. + +* Make a virtual environment: +``` +python -m venv venv +``` + +* Activate it + +- For Windows: +``` +venv\Scripts\activate +``` + +- For Linux/Mac/(other Unix): +``` +source venv/bin/activate +``` + +Once activated you can use regular python and pip commands and it will operate under your local project. + +You can verify this by running +``` +$ where python # for Windows +$ which pip python # for others +``` + +NOTE: This applies to not only pip and python but also other python scripts like black, pylint, etc + +After finishing your work you can simply use the ```deactivate``` command to deactivate your virtual environment and use the system's version of pip and python. + +Similarly closing the current terminal window also deactivates it. + + diff --git a/flask_final/__init__.py b/flask_final/__init__.py index 9714b7b..59ea944 100644 --- a/flask_final/__init__.py +++ b/flask_final/__init__.py @@ -23,7 +23,7 @@ def create_app(config): - app = Flask(__name__) + app = Flask(__name__.split('.')[0]) app.config.from_object(config) db.init_app(app) diff --git a/flask_final/config.py b/flask_final/config.py index 8fb0c33..ca58d7a 100644 --- a/flask_final/config.py +++ b/flask_final/config.py @@ -4,8 +4,7 @@ import json -class Config: - +class Prod: # basic debuggin properties: DEBUG = False TESTING = False @@ -15,51 +14,43 @@ class Config: SQLALCHEMY_TRACK_MODIFICATIONS = False SECRET_KEY = os.getenv("SECRET_KEY") - SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') - if('postgresql' not in SQLALCHEMY_DATABASE_URI): - SQLALCHEMY_DATABASE_URI = SQLALCHEMY_DATABASE_URI.replace("://", "ql://", 1) + SQLALCHEMY_DATABASE_URI = os.getenv("SQLALCHEMY_DATABASE_URI") or "" + SQLALCHEMY_DATABASE_URI = SQLALCHEMY_DATABASE_URI.replace( + # temporary workaround since sqlalchemy has deprecated postgres:// dialect and + # heroku hasn't updated or allowed users to change the default dialect + "postgres://", + "postgresql://", + 1, + ) # Email configs for reseting things you know. MAIL_SERVER = "smtp.googlemail.com" MAIL_PORT = 587 MAIL_USE_TLS = True - MAIL_USERNAME = os.getenv("EMAIL") - MAIL_PASSWORD = os.getenv("EMAIL_PASSWORD") - - -# check if there is a secrets.json file. -def Secrets(): - if os.path.exists("secrets.json"): - - with open("secrets.json", "r") as rf: - configs = json.load(rf) - - class Config_class(Config): - SECRET_KEY = configs["SECRET_KEY"] - MAIL_USERNAME = configs["MAIL_USERNAME"] - MAIL_PASSWORD = configs["MAIL_PASSWORD"] - SQLALCHEMY_DATABASE_URI = configs["SQLALCHEMY_DATABASE_URI"] - - return Config_class + MAIL_USERNAME = os.getenv("MAIL_USERNAME") + MAIL_PASSWORD = os.getenv("MAIL_PASSWORD") -class Debug(Config): +class Debug(Prod): DEBUG = True TESTING = True SQLALCHEMY_TRACK_MODIFICATIONS = (True,) -class SqliteDebug(Debug): - SQLALCHEMY_DATABASE_URI = "sqlite:///site.db" - - -class SqliteProduction(Config): - SQLALCHEMY_DATABASE_URI = "sqlite:///site.db" - - -class DatabaseDebug(Debug): - pass - - -class DatabaseProduction(Config): - pass +# check if there is a secrets.json file. +class Secrets(Debug): + def __init__(self): + file = "secrets.json" + if not os.path.exists(file): + print(f":: No secrets file {file} found! Exiting..") + sys.exit(1) + + with open(file, "r") as rf: + print(":: Reading secrets.json file (Only Debug mode suppported!)") + configs = json.load(rf) + + self.SECRET_KEY = configs["SECRET_KEY"] + self.MAIL_USERNAME = configs["MAIL_USERNAME"] + self.MAIL_PASSWORD = configs["MAIL_PASSWORD"] + self.SQLALCHEMY_DATABASE_URI = configs["SQLALCHEMY_DATABASE_URI"] + self.NEWS_API_KEY = configs.get("NEWS_API_KEY", "") diff --git a/manage.py b/manage.py index 9327457..4820f58 100644 --- a/manage.py +++ b/manage.py @@ -1,19 +1,22 @@ import os -import sys -from flask_final.config import DatabaseProduction, SqliteProduction + +from flask_final.config import Debug, Secrets from flask_final import db, create_app -args = sys.argv[1:] -app = create_app(DatabaseProduction) +is_env_var_set = os.getenv("SQLALCHEMY_DATABASE_URI") +if not is_env_var_set: + config = Secrets() +else: + config = Debug + +# Support for relative sqlite URIs +if config.SQLALCHEMY_DATABASE_URI == "sqlite:///site.db": + temp_app = create_app(config) + config.SQLALCHEMY_DATABASE_URI = "sqlite:///" + os.path.join( + temp_app.root_path, "site.db" + ) -# if database is not postgres but sqlite we initialize it diffrently. -if "sqlite" in args: - app = create_app(SqliteProduction) - print("setting sqlite db....") - with app.app_context(): - db.create_all() - print("done.") - sys.exit(0) +app = create_app(config) from flask_script import Manager from flask_migrate import Migrate, MigrateCommand diff --git a/run.py b/run.py index efde388..35c2a86 100644 --- a/run.py +++ b/run.py @@ -1,29 +1,27 @@ +import os import sys -from flask_final.config import ( - DatabaseProduction, - SqliteProduction, - DatabaseDebug, - SqliteDebug, - Secrets, -) -from flask_final import db, create_app +from flask_final.config import Prod, Debug, Secrets +from flask_final import create_app +app = None args = sys.argv[1:] if "run:app" in args: args = sys.argv[2:] -if len(args) == 0: - app = create_app(DatabaseProduction) +if len(args) == 0 or args[0] == "prod": + app = create_app(Prod) +elif args[0] == "debug": + app = create_app(Debug) else: - run_type = args[0] - config_map = { - "db-debug": DatabaseDebug, - "sqlite-prod": SqliteProduction, - "secrets": Secrets(), - "db-prod": DatabaseProduction, - "sqlite-debug": SqliteDebug, - } - app = create_app(config_map[run_type]) + print(f"Usage: run.py [prod/debug]. Unrecognized argument {args[0]}") + +is_env_var_set = os.getenv("SQLALCHEMY_DATABASE_URI") +if not is_env_var_set: + print( + ":: No environment variables set!\n" + ":: Falling back to Debug Mode and reading secrets from file" + ) + app = create_app(Secrets()) if __name__ == "__main__": app.run() diff --git a/template_secrets.json b/template_secrets.json index c4e456b..8649e43 100644 --- a/template_secrets.json +++ b/template_secrets.json @@ -1,6 +1,6 @@ { "SECRET_KEY": "jpt strings like lskdjf;slkdfjldskfj;kldf", + "SQLALCHEMY_DATABASE_URI": "sqlite:///site.db", "MAIL_USERNAME": "your email", - "MAIL_PASSWORD": "your password", - "SQLALCHEMY_DATABASE_URI": "sqlite:///site.db" + "MAIL_PASSWORD": "your password" }