Add transaction fetching support alexdlaird#20

amazonorders/cli.py

@@ -131,7 +131,7 @@ def history(ctx: Context,
 Order History for {year}{optional_start_index}{optional_full_details}
 -----------------------------------------------------------------------\n"""
                    .format(year=year,
-                           optional_start_index=optional_start_index,
+                optional_start_index=optional_start_index,
                            optional_full_details=optional_full_details))
         click.echo("Info: Fetching order history, this might take a minute ...")
 
@@ -177,6 +177,46 @@ def order(ctx: Context,
         ctx.fail(str(e))
 
 
+@amazon_orders_cli.command()
+@click.pass_context
+@click.option(
+    "--days",
+    default=365,
+    help="The number of days of transactions to get.",
+)
+def transactions(ctx: Context, **kwargs: Any):
+    """
+    Retrieve Amazon order history for a given year.
+    """
+    amazon_session = ctx.obj["amazon_session"]
+
+    try:
+        _authenticate(ctx, amazon_session)
+
+        days = kwargs["days"]
+
+        click.echo(
+            """-----------------------------------------------------------------------
+Transaction History for {days} days
+-----------------------------------------------------------------------\n""".format(
+                days=days
+            )
+        )
+        click.echo("Info: Fetching transaction history, this might take a minute ...")
+
+        amazon_transactions = AmazonOrders(amazon_session, config=ctx.obj["conf"])
+
+        transactions = amazon_transactions.get_transactions(days=days)
+
+        click.echo("... {} transactions parsed.\n".format(len(transactions)))
+
+        for transaction in transactions:
+            click.echo(f"{transaction}\n")
+    except AmazonOrdersError as e:
+        logger.debug("An error occurred.", exc_info=True)
+        ctx.fail(str(e))
+
+
 @amazon_orders_cli.command(short_help="Check if persisted session exists.")
 @click.pass_context
 def check_session(ctx: Context):

amazonorders/constants.py

@@ -20,21 +20,30 @@ class Constants:
     SIGN_OUT_URL = f"{BASE_URL}/gp/sign-out.html"
 
     ##########################################################################
-    # URLs for AmazonOrders
+    # URLs for orders
     ##########################################################################
 
     ORDER_HISTORY_LANDING_URL = f"{BASE_URL}/gp/css/order-history"
     ORDER_HISTORY_URL = f"{BASE_URL}/your-orders/orders"
     ORDER_DETAILS_URL = f"{BASE_URL}/gp/your-account/order-details"
     HISTORY_FILTER_QUERY_PARAM = "timeFilter"
 
+    ##########################################################################
+    # URLs for transactions
+    ##########################################################################
+
+    TRANSACTION_HISTORY_LANDING_ROUTE = "/cpe/yourpayments/transactions"
+    TRANSACTION_HISTORY_LANDING_URL = f"{BASE_URL}{TRANSACTION_HISTORY_LANDING_ROUTE}"
+
+    TRANSACTION_DATE_FORMAT = "%B %d, %Y"
+
     ##########################################################################
     # Headers
     ##########################################################################
 
     BASE_HEADERS = {
         "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,"
-                  "application/signed-exchange;v=b3;q=0.7",
+        "application/signed-exchange;v=b3;q=0.7",
         "Accept-Encoding": "gzip, deflate, br",
         "Accept-Language": "en-US,en;q=0.9",
         "Cache-Control": "max-age=0",
@@ -51,5 +60,5 @@ class Constants:
         "Sec-Fetch-User": "?1",
         "Viewport-Width": "1393",
         "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) "
-                      "Chrome/120.0.0.0 Safari/537.36",
+        "Chrome/120.0.0.0 Safari/537.36",
     }

amazonorders/entity/parsable.py

@@ -2,13 +2,14 @@
 __license__ = "MIT"
 
 import logging
+import re
 from typing import Any, Callable, Optional, Type, Union
 
 from bs4 import Tag
 
 from amazonorders import util
 from amazonorders.conf import AmazonOrdersConfig
-from amazonorders.exception import AmazonOrderEntityError, AmazonOrdersError
+from amazonorders.exception import AmazonOrdersEntityError, AmazonOrdersError
 
 logger = logging.getLogger(__name__)
 
@@ -117,7 +118,7 @@ def simple_parse(self,
                 break
 
         if not value and required:
-            raise AmazonOrderEntityError(
+            raise AmazonOrdersEntityError(
                 "When building {name}, field for selector `{selector}` was None, but this is not allowed.".format(
                     name=self.__class__.__name__, selector=selector))
 
@@ -160,9 +161,9 @@ def to_currency(self,
         if not value:
             return None
 
-        currency = util.to_type(
-            value.strip().replace("$", "").replace(",", "")
-        )
+        value = value.strip()
+        value = re.sub("[a-zA-Z$,]+", "", value)
+        currency = util.to_type(value)
 
         if isinstance(currency, str):
             return None

amazonorders/entity/transaction.py

@@ -0,0 +1,58 @@
+__copyright__ = "Copyright (c) 2024 Jeff Sawatzky"
+__license__ = "MIT"
+
+import logging
+import re
+from datetime import date
+
+from bs4 import Tag
+
+from amazonorders.conf import AmazonOrdersConfig
+from amazonorders.entity.parsable import Parsable
+
+logger = logging.getLogger(__name__)
+
+
+class Transaction(Parsable):
+    """
+    An Amazon Transaction
+    """
+
+    def __init__(self, parsed: Tag, config: AmazonOrdersConfig, completed_date: date) -> None:
+        super().__init__(parsed, config)
+
+        #: The Transaction completed date.
+        self.completed_date: date = completed_date
+        #: The Transaction payment method.
+        self.payment_method: str = self.safe_simple_parse(
+            selector=self.config.selectors.FIELD_TRANSACTION_PAYMENT_METHOD_SELECTOR
+        )
+        #: The Transaction grand total.
+        self.grand_total: float = self.safe_parse(self._parse_grand_total)
+        #: The Transaction was a refund or not.
+        self.is_refund: bool = self.grand_total > 0
+        #: The Transaction order number.
+        self.order_number: str = self.safe_parse(self._parse_order_number)
+        #: The Transaction seller name.
+        self.seller: str = self.safe_simple_parse(
+            selector=self.config.selectors.FIELD_TRANSACTION_SELLER_NAME_SELECTOR
+        )
+
+    def __repr__(self) -> str:
+        return f'<Transaction {self.completed_date}: "Order #{self.order_number}", "Grand Total {self.grand_total}">'
+
+    def __str__(self) -> str:  # pragma: no cover
+        return f"Transaction {self.completed_date}: Order #{self.order_number}, Grand Total {self.grand_total}"
+
+    def _parse_grand_total(self) -> float:
+        value = self.simple_parse(self.config.selectors.FIELD_TRANSACTION_GRAND_TOTAL_SELECTOR)
+        value = self.to_currency(value)
+
+        return value
+
+    def _parse_order_number(self) -> str:
+        value = self.simple_parse(self.config.selectors.FIELD_TRANSACTION_ORDER_NUMBER_SELECTOR)
+        match = re.match(".*#([0-9-]+)$", value)
+        value = match.group(1) if match else ""
+
+        return value

amazonorders/exception.py

@@ -23,7 +23,7 @@ class AmazonOrdersAuthError(AmazonOrdersError):
     pass
 
 
-class AmazonOrderEntityError(AmazonOrdersError):
+class AmazonOrdersEntityError(AmazonOrdersError):
     """
     Raised when an ``amazon-orders`` entity parsing error has occurred.
     """

amazonorders/lib/transactions.py

@@ -0,0 +1,62 @@
+__copyright__ = "Copyright (c) 2024 Jeff Sawatzky"
+__license__ = "MIT"
+
+import datetime
+from typing import Dict, List, Optional, Tuple
+
+from bs4 import Tag
+
+from amazonorders.conf import AmazonOrdersConfig
+from amazonorders.entity.transaction import Transaction
+
+
+def parse_transaction_form_tag(
+    form_tag: Tag, config: AmazonOrdersConfig
+) -> Tuple[List[Transaction], Optional[str], Optional[Dict[str, str]]]:
+    transactions = []
+    date_container_tags = form_tag.select(
+        config.selectors.TRANSACTION_DATE_CONTAINERS_SELECTOR
+    )
+    for date_container_tag in date_container_tags:
+        date_tag = date_container_tag.select_one(
+            config.selectors.FIELD_TRANSACTION_COMPLETED_DATE_SELECTOR
+        )
+        assert date_tag is not None
+
+        date_str = date_tag.text
+        date = datetime.datetime.strptime(
+            date_str, config.constants.TRANSACTION_DATE_FORMAT
+        ).date()
+
+        transactions_container_tag = date_container_tag.find_next_sibling(
+            config.selectors.TRANSACTIONS_CONTAINER_SELECTOR
+        )
+        assert isinstance(transactions_container_tag, Tag)
+
+        transaction_tags = transactions_container_tag.select(
+            config.selectors.TRANSACTIONS_SELECTOR
+        )
+        for transaction_tag in transaction_tags:
+            transaction = Transaction(transaction_tag, config, date)
+            transactions.append(transaction)
+
+    form_state_input = form_tag.select_one(
+        config.selectors.TRANSACTIONS_NEXT_PAGE_INPUT_STATE_SELECTOR
+    )
+    form_ie_input = form_tag.select_one(
+        config.selectors.TRANSACTIONS_NEXT_PAGE_INPUT_IE_SELECTOR
+    )
+    next_page_input = form_tag.select_one(
+        config.selectors.TRANSACTIONS_NEXT_PAGE_INPUT_SELECTOR
+    )
+    if not next_page_input or not form_state_input or not form_ie_input:
+        return (transactions, None, None)
+
+    next_page_post_url = str(form_tag["action"])
+    next_page_post_data = {
+        "ppw-widgetState": str(form_state_input["value"]),
+        "ie": str(form_ie_input["value"]),
+        str(next_page_input["name"]): "",
+    }
+
+    return (transactions, next_page_post_url, next_page_post_data)

amazonorders/orders.py

@@ -8,7 +8,9 @@
 from amazonorders import util
 from amazonorders.conf import AmazonOrdersConfig
 from amazonorders.entity.order import Order
+from amazonorders.entity.transaction import Transaction
 from amazonorders.exception import AmazonOrdersError, AmazonOrdersNotFoundError
+from amazonorders.lib.transactions import parse_transaction_form_tag
 from amazonorders.session import AmazonSession
 
 logger = logging.getLogger(__name__)
@@ -117,3 +119,48 @@ def get_order(self,
         order = self.config.order_cls(order_details_tag, self.config, full_details=True)
 
         return order
+
+    def get_transactions(
+        self,
+        days: int = 365,
+    ) -> List[Transaction]:
+        """
+        Get the Amazon transactions for the given number of days.
+
+        :param days: The number of days worth of transactions to get.
+        :return: A list of the requested Transactions.
+        """
+        if not self.amazon_session.is_authenticated:
+            raise AmazonOrdersError("Call AmazonSession.login() to authenticate first.")
+
+        min_date = datetime.date.today() - datetime.timedelta(days=days)
+
+        self.amazon_session.get(self.config.constants.TRANSACTION_HISTORY_LANDING_URL)
+        assert self.amazon_session.last_response_parsed
+
+        form_tag = self.amazon_session.last_response_parsed.select_one(
+            self.config.selectors.TRANSACTION_HISTORY_FORM_SELECTOR
+        )
+
+        transactions: List[Transaction] = []
+        while form_tag:
+            loaded_transactions, next_page_post_url, next_page_post_data = (
+                parse_transaction_form_tag(form_tag, self.config)
+            )
+            for transaction in loaded_transactions:
+                if transaction.completed_date >= min_date:
+                    transactions.append(transaction)
+                else:
+                    return transactions
+
+            if next_page_post_url is None:
+                return transactions
+
+            self.amazon_session.post(next_page_post_url, data=next_page_post_data)
+            assert self.amazon_session.last_response_parsed
+
+            form_tag = self.amazon_session.last_response_parsed.select_one(
+                self.config.selectors.TRANSACTION_HISTORY_FORM_SELECTOR
+            )
+
+        return transactions

amazonorders/selectors.py

@@ -1,6 +1,8 @@
 __copyright__ = "Copyright (c) 2024 Alex Laird"
 __license__ = "MIT"
 
+from amazonorders.constants import Constants
+
 
 class Selectors:
     ##########################################################################
@@ -95,3 +97,32 @@ class Selectors:
 
     FIELD_SELLER_NAME_SELECTOR = ["a", "span"]
     FIELD_SELLER_LINK_SELECTOR = "a"
+
+    #####################################
+    # CSS selectors for Transaction fields
+    #####################################
+
+    TRANSACTION_HISTORY_FORM_SELECTOR = f"form[method='post'][action$='{Constants.TRANSACTION_HISTORY_LANDING_ROUTE}']"
+    TRANSACTION_DATE_CONTAINERS_SELECTOR = "div.apx-transaction-date-container"
+    TRANSACTIONS_CONTAINER_SELECTOR = "div"
+    TRANSACTIONS_SELECTOR = "div.apx-transactions-line-item-component-container"
+
+    TRANSACTIONS_NEXT_PAGE_INPUT_SELECTOR = (
+        "input[type='submit'][name^='ppw-widgetEvent:DefaultNextPageNavigationEvent']"
+    )
+    TRANSACTIONS_NEXT_PAGE_INPUT_STATE_SELECTOR = "input[name='ppw-widgetState']"
+    TRANSACTIONS_NEXT_PAGE_INPUT_IE_SELECTOR = "input[name='ie']"
+
+    FIELD_TRANSACTION_COMPLETED_DATE_SELECTOR = "span"
+    FIELD_TRANSACTION_PAYMENT_METHOD_SELECTOR = (
+        "div.apx-transactions-line-item-component-container > div:nth-child(1) span.a-size-base"
+    )
+    FIELD_TRANSACTION_GRAND_TOTAL_SELECTOR = (
+        "div.apx-transactions-line-item-component-container > div:nth-child(1) span.a-size-base-plus"
+    )
+    FIELD_TRANSACTION_ORDER_NUMBER_SELECTOR = (
+        "div.apx-transactions-line-item-component-container > div:nth-child(2) a.a-link-normal"
+    )
+    FIELD_TRANSACTION_SELLER_NAME_SELECTOR = (
+        "div.apx-transactions-line-item-component-container > div:nth-child(3) span.a-size-base"
+    )

docs/api.rst

@@ -65,6 +65,11 @@ Entities
     :private-members:
     :show-inheritance:
 
+.. automodule:: amazonorders.entity.transaction
+    :members:
+    :private-members:
+    :show-inheritance:
+
 Exceptions
 ----------
 

pyproject.toml

@@ -36,6 +36,7 @@ dev = [
     "coverage[toml]",
     "flake8",
     "flake8-pyproject",
+    "freezegun",
     "pep8-naming",
     "responses",
     "flask",
@@ -107,4 +108,4 @@ python_version = "3.8"
 module = [
     "amazoncaptcha.*",
 ]
-ignore_missing_imports = true
+ignore_missing_imports = true