WIP - v5

.env.example

@@ -0,0 +1,8 @@
+SQLALCHEMY_DATABASE_URI=postgresql+psycopg://atopile:changeme@localhost/atopile-components
+
+POSTGRES_USER=atopile
+POSTGRES_PASSWORD=my-password
+POSTGRES_DB=atopile-components
+
+JLCPCB_KEY="app_some_key"
+JLCPCB_SECRET="app_some_secret"

.gitignore

@@ -25,4 +25,174 @@ node_modules/
 *.egg-info/
 
 # Ignore virtual environment
-venv/
+venv/
+
+##### Python.gitignore #####
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+
+#### Additional things ####
+# sqlite databases
+example.db

Dockerfile.scraper

@@ -0,0 +1,10 @@
+FROM python:3.12
+
+WORKDIR /usr/src/app
+
+COPY requirements-scraper.txt ./
+RUN pip install --no-cache-dir -r requirements-scraper.txt
+
+COPY . .
+
+CMD [ "python", "./src/scraper.py" ]

Dockerfile → Dockerfile.server

@@ -7,4 +7,4 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . .
 
-CMD [ "python", "./jlcpcb_scraper/main.py" ]
+CMD [ "python", "./src/server.py" ]

README.md

@@ -12,6 +12,9 @@ pip install -r requirements.txt
 
 ## Usage
 
+Fill in the details in the .env file
+
+### OR
 
 Create a Postgres DB and make sure it is reachable by the pc that will execute this script.
 Set environment variable SQLALCHEMY_DATABASE_URI, JLCPCB_KEY and JLCPCB_SECRET
@@ -24,7 +27,7 @@ export JLCPCB_SECRET=<SECRET>
 Execute main.py
 ```
 python3 jlcpcb_scraper/main.py
-``` 
+```
 
 
 ## Testing TODO, INCOMPLETE
@@ -43,4 +46,56 @@ Please make sure to update tests as appropriate.
 
 ## License
 
-[MIT](https://choosealicense.com/licenses/mit/)
+[MIT](https://choosealicense.com/licenses/mit/)
+
+## Creating a DB
+
+You need Postgres running; eg. `docker up` from the component server project
+
+Alembic won't create a DB, you need to do that manually from Postgres Admin or the likes. Name is `atopile-components` or something like that.
+
+I've had issues with alembic creating tables from scratch? Perhaps I had things in a dirty state.
+
+```python
+from sqlalchemy import create_engine
+
+from jlcpcb_scraper.config import config
+from jlcpcb_scraper.models import Base
+
+engine = create_engine(config.SQLALCHEMY_DATABASE_URI)
+Base.metadata.create_all(engine)
+```
+
+## Local developement
+
+### Running a local postgreSQL database
+
+Make sure docker is running and run `docker-compose up`.
+
+### Updating the alembic database schema
+
+If you change the database schema, use the following command to update the alembic version and migration script: `alembic revision --autogenerate -m "explain what happened"`
+
+To apply the changes to the database, run `alembic upgrade head`. Your database should now follow the alchemy ORM schema.
+
+### Inspecting your database
+
+To inspect the contents of your database and make changes manually, use the `psql` util. `psql` can be installed with `brew` on mac with `brew install postgresql@16`.
+
+```bash
+psql -U atopile -h localhost -p 5432 -d atopile-components
+```
+
+To list the tables, invoke `\dt`. To quit, invoke `exit`. To delete tables we are not using anymore, invoke `DROP TABLE public.alembic_version, public.ranged_values;
+
+### Starting the server
+
+1. Run `docker-compose up` to start PostgreSQL.
+2. Run `fastapi dev endpoints.py` to start the FastAPI server.
+
+
+## Up-revving database
+
+1. Upgrade the schema
+2. Create a revision `alembic revision --autogenerate -m "Why?"`
+3. Run the upgrade on the `alembic upgrade head`

alembic.ini

@@ -82,7 +82,7 @@ version_path_separator = os  # Use os.pathsep. Default configuration used for ne
 
 # Logging configuration
 [loggers]
-keys = root,sqlalchemy,alembic
+keys = root,sqlalchemy,alembic,component_server
 
 [handlers]
 keys = console
@@ -91,7 +91,7 @@ keys = console
 keys = generic
 
 [logger_root]
-level = WARN
+level = DEBUG
 handlers = console
 qualname =
 
@@ -105,6 +105,11 @@ level = INFO
 handlers =
 qualname = alembic
 
+[logger_component_server]
+level = INFO
+handlers =
+qualname = component_server
+
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)

alembic/env.py

@@ -5,8 +5,8 @@
 
 from alembic import context
 
-from jlcpcb_scraper.config import config as app_config
-from jlcpcb_scraper.base import *
+from component_server.config import config as app_config
+from component_server.base import *
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.

alembic/versions/0ad54fb43dd5_add_inductor.py

@@ -0,0 +1,51 @@
+"""Add inductor
+
+Revision ID: 0ad54fb43dd5
+Revises: a2e2efe545fd
+Create Date: 2024-07-10 17:23:21.012278
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '0ad54fb43dd5'
+down_revision: Union[str, None] = 'a2e2efe545fd'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('inductor',
+    sa.Column('inductance_henries_min', sa.Float(), nullable=True),
+    sa.Column('inductance_henries_max', sa.Float(), nullable=True),
+    sa.Column('dc_resistance_min', sa.Float(), nullable=True),
+    sa.Column('dc_resistance_max', sa.Float(), nullable=True),
+    sa.Column('operating_current_amps_min', sa.Float(), nullable=True),
+    sa.Column('operating_current_amps_max', sa.Float(), nullable=True),
+    sa.Column('operating_temp_celsius_min', sa.Float(), nullable=True),
+    sa.Column('operating_temp_celsius_max', sa.Float(), nullable=True),
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('last_update', sa.DateTime(), nullable=True),
+    sa.Column('price', sa.Float(), nullable=True),
+    sa.Column('stock', sa.Integer(), nullable=True),
+    sa.Column('overhead_cost', sa.Float(), nullable=True),
+    sa.Column('rating', sa.Integer(), nullable=True),
+    sa.Column('lcsc_id', sa.String(), nullable=True),
+    sa.Column('mpn', sa.String(), nullable=True),
+    sa.Column('package', sa.String(), nullable=True),
+    sa.Column('footprint_name', sa.String(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('lcsc_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('inductor')
+    # ### end Alembic commands ###