From 48fe4f8baf22adc09bd726fe45aa9c88b17a7f82 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Mon, 4 Nov 2019 19:44:10 +0100 Subject: [PATCH 1/4] Workaround eth-* not building on F-Droid, closes #167 F-Droid build server doesn't have Python3.6. Hence making recipes would force the build process to pick up the compiled host Python. Note that we had to pull `libssl-dev` system dependency for the build to pass now, refs https://github.com/kivy/python-for-android/issues/1859 --- Makefile | 2 +- .../recipes/eth-abi/__init__.py | 20 ++++++++++++++++++ .../recipes/eth-account/__init__.py | 21 +++++++++++++++++++ .../recipes/eth-account/setup.py.patch | 12 +++++++++++ .../recipes/web3/__init__.py | 20 ++++++++++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 src/python-for-android/recipes/eth-abi/__init__.py create mode 100644 src/python-for-android/recipes/eth-account/__init__.py create mode 100644 src/python-for-android/recipes/eth-account/setup.py.patch create mode 100644 src/python-for-android/recipes/web3/__init__.py diff --git a/Makefile b/Makefile index 4b17e94..ca3d2ab 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ VIRTUAL_ENV ?= venv ACTIVATE_PATH=$(VIRTUAL_ENV)/bin/activate PIP=$(VIRTUAL_ENV)/bin/pip -TOX=`which tox` PYTHON_MAJOR_VERSION=3 PYTHON_MINOR_VERSION=7 PYTHON_VERSION=$(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION) @@ -34,6 +33,7 @@ SYSTEM_DEPENDENCIES_ANDROID= \ libffi-dev \ libltdl-dev \ git \ + libssl-dev \ libtool \ openjdk-8-jdk-headless \ patch \ diff --git a/src/python-for-android/recipes/eth-abi/__init__.py b/src/python-for-android/recipes/eth-abi/__init__.py new file mode 100644 index 0000000..5c4b326 --- /dev/null +++ b/src/python-for-android/recipes/eth-abi/__init__.py @@ -0,0 +1,20 @@ +from pythonforandroid.recipe import PythonRecipe + + +class EthAbiRecipe(PythonRecipe): + """ + This recipes is a workaround to build on F-Droid build server. + Their build server doesn't have Python3.6 yet, hence we want to force this + recipe to build on the compiled host Python, refs: + https://github.com/AndreMiras/EtherollApp/issues/167 + """ + version = '2.0.0' + url = ( + 'https://pypi.python.org/packages/source/e/eth-abi/' + 'eth-abi-{version}.tar.gz' + ) + depends = ['setuptools'] + call_hostpython_via_targetpython = False + + +recipe = EthAbiRecipe() diff --git a/src/python-for-android/recipes/eth-account/__init__.py b/src/python-for-android/recipes/eth-account/__init__.py new file mode 100644 index 0000000..bda7ef9 --- /dev/null +++ b/src/python-for-android/recipes/eth-account/__init__.py @@ -0,0 +1,21 @@ +from pythonforandroid.recipe import PythonRecipe + + +class EthAccountRecipe(PythonRecipe): + """ + This recipes is a workaround to build on F-Droid build server. + Their build server doesn't have Python3.6 yet, hence we want to force this + recipe to build on the compiled host Python, refs: + https://github.com/AndreMiras/EtherollApp/issues/167 + """ + version = '0.4.0' + url = ( + 'https://pypi.python.org/packages/source/e/eth-account/' + 'eth-account-{version}.tar.gz' + ) + depends = ['setuptools'] + patches = ['setup.py.patch'] + call_hostpython_via_targetpython = False + + +recipe = EthAccountRecipe() diff --git a/src/python-for-android/recipes/eth-account/setup.py.patch b/src/python-for-android/recipes/eth-account/setup.py.patch new file mode 100644 index 0000000..4ba159f --- /dev/null +++ b/src/python-for-android/recipes/eth-account/setup.py.patch @@ -0,0 +1,12 @@ +diff --git a/setup.py b/setup.py +index f4a2ad0..4e52b68 100644 +--- a/setup.py ++++ b/setup.py +@@ -57,7 +57,6 @@ setup( + "hexbytes>=0.1.0,<1", + "rlp>=1.0.0,<2" + ], +- setup_requires=['setuptools-markdown'], + python_requires='>=3.6, <4', + extras_require=extras_require, + py_modules=['eth_account'], diff --git a/src/python-for-android/recipes/web3/__init__.py b/src/python-for-android/recipes/web3/__init__.py new file mode 100644 index 0000000..78f3979 --- /dev/null +++ b/src/python-for-android/recipes/web3/__init__.py @@ -0,0 +1,20 @@ +from pythonforandroid.recipe import PythonRecipe + + +class Web3Recipe(PythonRecipe): + """ + This recipes is a workaround to build on F-Droid build server. + Their build server doesn't have Python3.6 yet, hence we want to force this + recipe to build on the compiled host Python, refs: + https://github.com/AndreMiras/EtherollApp/issues/167 + """ + version = 'web3==5.2.0' + url = ( + 'https://pypi.python.org/packages/source/w/web3/' + 'web3-{version}.tar.gz' + ) + depends = ['setuptools'] + call_hostpython_via_targetpython = False + + +recipe = Web3Recipe() From 5cef3275cb90b574ea5f0008def443378626aee0 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Fri, 8 Nov 2019 19:13:23 +0100 Subject: [PATCH 2/4] Bumps to pyetheroll==20191108 Improves API key management and fixes user agent ban on Ropsten. - fixes #156 - fixes #157 --- buildozer.spec | 2 +- requirements.txt | 2 +- src/etherollapp/CHANGELOG.md | 8 +++- src/etherollapp/api_key.json | 1 + src/etherollapp/etheroll/controller.py | 8 ++-- src/etherollapp/etheroll/utils.py | 36 ++++++++++++++++ src/etherollapp/service/main.py | 8 ++-- src/etherollapp/tests/etheroll/__init__.py | 0 src/etherollapp/tests/etheroll/test_utils.py | 45 ++++++++++++++++++++ src/etherollapp/tests/service/test_main.py | 7 +-- 10 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 src/etherollapp/api_key.json create mode 100644 src/etherollapp/tests/etheroll/__init__.py create mode 100644 src/etherollapp/tests/etheroll/test_utils.py diff --git a/buildozer.spec b/buildozer.spec index fa8732b..f42ad6f 100644 --- a/buildozer.spec +++ b/buildozer.spec @@ -70,7 +70,7 @@ requirements = Pillow==5.2.0, plyer==1.3.1, pycryptodome==3.4.6, - pyetheroll==20191018, + pyetheroll==20191108, Pygments==2.2.0, python3==3.7.1, pyzbar==0.1.8, diff --git a/requirements.txt b/requirements.txt index 06c572c..51746da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ mypy==0.730 oscpy==0.3.0 Pillow==5.2.0 plyer==1.3.1 -pyetheroll==20191018 +pyetheroll==20191108 pytest pyzbar==0.1.8 qrcode==6.0 diff --git a/src/etherollapp/CHANGELOG.md b/src/etherollapp/CHANGELOG.md index a8fcfae..5eea3b7 100644 --- a/src/etherollapp/CHANGELOG.md +++ b/src/etherollapp/CHANGELOG.md @@ -1,9 +1,15 @@ # Change Log +## [Unreleased] + + - Fix 403 errors on Ropsten, refs #156, #157 + - Fix broken F-Droid build, refs #167 + + ## [v2019.1101] - - Bumps Kivy related dependencies + - Bump Kivy related dependencies - Build and CI improvements - Transfer out feature, refs #80, #105 - Migrate to NDK 19b, refs #154 diff --git a/src/etherollapp/api_key.json b/src/etherollapp/api_key.json new file mode 100644 index 0000000..38f63fc --- /dev/null +++ b/src/etherollapp/api_key.json @@ -0,0 +1 @@ +{ "key" : "E9K4A1AC8H1V3ZIR1DAIKZ6B961CRXF2DR" } diff --git a/src/etherollapp/etheroll/controller.py b/src/etherollapp/etheroll/controller.py index 59542f0..fccaec8 100755 --- a/src/etherollapp/etheroll/controller.py +++ b/src/etherollapp/etheroll/controller.py @@ -17,7 +17,7 @@ from etherollapp.etheroll.settings_screen import SettingsScreen from etherollapp.etheroll.switchaccount import SwitchAccountScreen from etherollapp.etheroll.ui_utils import Dialog, load_kv_from_py -from etherollapp.etheroll.utils import run_in_thread +from etherollapp.etheroll.utils import get_etherscan_api_key, run_in_thread from etherollapp.osc.osc_app_server import OscAppServer from etherollapp.sentry_utils import configure_sentry from etherollapp.service.utils import start_roll_polling_service @@ -38,7 +38,6 @@ def __init__(self, **kwargs): self.disabled = True Clock.schedule_once(self._after_init) self._account_passwords = {} - self._pyetheroll = None def _after_init(self, dt): """Inits pyethapp and binds events.""" @@ -82,9 +81,8 @@ def pyetheroll(self): """ from pyetheroll.etheroll import Etheroll chain_id = Settings.get_stored_network() - if self._pyetheroll is None or self._pyetheroll.chain_id != chain_id: - self._pyetheroll = Etheroll(API_KEY_PATH, chain_id) - return self._pyetheroll + api_key = get_etherscan_api_key(API_KEY_PATH) + return Etheroll.get_or_create(api_key, chain_id) @property def account_utils(self): diff --git a/src/etherollapp/etheroll/utils.py b/src/etherollapp/etheroll/utils.py index db6676f..9fce6b3 100644 --- a/src/etherollapp/etheroll/utils.py +++ b/src/etherollapp/etheroll/utils.py @@ -1,8 +1,13 @@ +import json +import logging +import os import threading from io import StringIO from kivy.utils import platform +logger = logging.getLogger(__name__) + def run_in_thread(fn): """ @@ -48,6 +53,37 @@ def check_request_write_permission(): return had_permission +def get_etherscan_api_key(api_key_path: str = None) -> str: + """ + Tries to retrieve etherscan API key from path or from environment. + The files content should be in the form: + ```json + { "key" : "YourApiKeyToken" } + ``` + """ + DEFAULT_API_KEY_TOKEN = "YourApiKeyToken" + etherscan_api_key = os.environ.get("ETHERSCAN_API_KEY") + if etherscan_api_key is not None: + return etherscan_api_key + elif api_key_path is None: + logger.warning( + "Cannot get Etherscan API key. " + f"No path provided, defaulting to {DEFAULT_API_KEY_TOKEN}." + ) + return DEFAULT_API_KEY_TOKEN + else: + try: + with open(api_key_path, mode="r") as key_file: + etherscan_api_key = json.loads(key_file.read())["key"] + except FileNotFoundError: + logger.warning( + f"Cannot get Etherscan API key. File {api_key_path} not found," + f" defaulting to {DEFAULT_API_KEY_TOKEN}." + ) + return DEFAULT_API_KEY_TOKEN + return etherscan_api_key + + class StringIOCBWrite(StringIO): """Inherits StringIO, provides callback on write.""" diff --git a/src/etherollapp/service/main.py b/src/etherollapp/service/main.py index cb5165b..51fdff0 100755 --- a/src/etherollapp/service/main.py +++ b/src/etherollapp/service/main.py @@ -27,6 +27,7 @@ from etherollapp.ethereum_utils import AccountUtils from etherollapp.etheroll.constants import API_KEY_PATH from etherollapp.etheroll.settings import Settings +from etherollapp.etheroll.utils import get_etherscan_api_key from etherollapp.osc.osc_app_client import OscAppClient from etherollapp.sentry_utils import configure_sentry @@ -71,7 +72,6 @@ def __init__(self, osc_server_port=None): """ Set `osc_server_port` to enable UI synchronization with service. """ - self._pyetheroll = None self._account_utils = None # per address cached merged logs, used to compare with next pulls self.merged_logs = {} @@ -122,10 +122,8 @@ def pyetheroll(self): Also recreates the object if the chain_id changed. """ chain_id = Settings.get_stored_network() - print(f'chain_id: {chain_id}') - if self._pyetheroll is None or self._pyetheroll.chain_id != chain_id: - self._pyetheroll = Etheroll(API_KEY_PATH, chain_id) - return self._pyetheroll + api_key = get_etherscan_api_key(API_KEY_PATH) + return Etheroll.get_or_create(api_key, chain_id) def pull_account_rolls(self, account): """ diff --git a/src/etherollapp/tests/etheroll/__init__.py b/src/etherollapp/tests/etheroll/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/etherollapp/tests/etheroll/test_utils.py b/src/etherollapp/tests/etheroll/test_utils.py new file mode 100644 index 0000000..6db62f0 --- /dev/null +++ b/src/etherollapp/tests/etheroll/test_utils.py @@ -0,0 +1,45 @@ +from unittest import mock + +from etherollapp.etheroll.utils import get_etherscan_api_key + + +class TestUtils: + def test_get_etherscan_api_key(self): + """ + Verifies the key can be retrieved from either: + 1) environment + 2) file + 3) or fallbacks on default key + """ + expected_key = "0102030405060708091011121314151617" + # 1) environment + with mock.patch.dict( + "os.environ", {"ETHERSCAN_API_KEY": expected_key} + ): + actual_key = get_etherscan_api_key() + assert actual_key == expected_key + # 2) file + read_data = '{ "key" : "%s" }' % (expected_key) + api_key_path = "api_key.json" + with mock.patch( + "builtins.open", mock.mock_open(read_data=read_data) + ) as m_open: + actual_key = get_etherscan_api_key(api_key_path=api_key_path) + assert expected_key == actual_key + # verifies the file was read + assert m_open.call_args_list == [mock.call(api_key_path, mode="r")] + # 3) or fallbacks on default key + with mock.patch("builtins.open") as m_open, mock.patch( + "etherollapp.etheroll.utils.logger" + ) as m_logger: + m_open.side_effect = FileNotFoundError + actual_key = get_etherscan_api_key(api_key_path) + assert "YourApiKeyToken" == actual_key + # verifies the fallback warning was logged + assert m_logger.warning.call_args_list == [ + mock.call( + "Cannot get Etherscan API key. " + "File api_key.json not found, " + "defaulting to YourApiKeyToken." + ) + ] diff --git a/src/etherollapp/tests/service/test_main.py b/src/etherollapp/tests/service/test_main.py index 7774f5a..76b22a4 100644 --- a/src/etherollapp/tests/service/test_main.py +++ b/src/etherollapp/tests/service/test_main.py @@ -5,6 +5,7 @@ from kivy.app import App from pyetheroll.constants import ChainID +from pyetheroll.etheroll import Etheroll from etherollapp.service.main import EtherollApp, MonitorRollsService @@ -119,19 +120,19 @@ def test_pyetheroll(self): The cached value should be updated on (testnet/mainnet) network change. """ service = MonitorRollsService() - assert service._pyetheroll is None + # deletes the cached property eventually initialized by other tests + Etheroll._etheroll = None with tempfile.TemporaryDirectory() as temp_path, \ patch_get_abi() as m_get_abi, \ mock.patch.dict('os.environ', {'XDG_CONFIG_HOME': temp_path}): assert service.pyetheroll is not None assert m_get_abi.mock_calls == [mock.call()] - assert service._pyetheroll is not None pyetheroll = service.pyetheroll # it's obviously pointing to the same object for now, # but shouldn't not be later after we update some settings assert pyetheroll == service.pyetheroll assert pyetheroll.chain_id == ChainID.MAINNET - # the cached pyetheroll object is invalidated if the network changes + # the cached object is invalidated if the network changes with mock.patch( 'etherollapp.service.main.Settings.get_stored_network' ) as m_get_stored_network, patch_get_abi() as m_get_abi: From c064fd0416dcad281c0f8ecaddc8bca010884f4e Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Fri, 8 Nov 2019 20:09:34 +0100 Subject: [PATCH 3/4] Fixes Kivy low FPS performance issue refs: - https://github.com/kivy/kivy/pull/6242 - https://github.com/kivy/python-for-android/issues/2002 --- src/etherollapp/CHANGELOG.md | 1 + .../recipes/kivy/__init__.py | 64 +++++++++++++++++++ .../recipes/kivy/revert4219.patch | 37 +++++++++++ 3 files changed, 102 insertions(+) create mode 100644 src/python-for-android/recipes/kivy/__init__.py create mode 100644 src/python-for-android/recipes/kivy/revert4219.patch diff --git a/src/etherollapp/CHANGELOG.md b/src/etherollapp/CHANGELOG.md index 5eea3b7..bffa2e3 100644 --- a/src/etherollapp/CHANGELOG.md +++ b/src/etherollapp/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] + - Fix Kivy low FPS performances issue - Fix 403 errors on Ropsten, refs #156, #157 - Fix broken F-Droid build, refs #167 diff --git a/src/python-for-android/recipes/kivy/__init__.py b/src/python-for-android/recipes/kivy/__init__.py new file mode 100644 index 0000000..3784d6a --- /dev/null +++ b/src/python-for-android/recipes/kivy/__init__.py @@ -0,0 +1,64 @@ +import glob +from os.path import basename, exists, join + +import sh +from pythonforandroid.recipe import CythonRecipe +from pythonforandroid.toolchain import current_directory, shprint + + +class KivyRecipe(CythonRecipe): + version = '1.11.1' + url = 'https://github.com/kivy/kivy/archive/{version}.zip' + name = 'kivy' + + depends = ['sdl2', 'pyjnius', 'setuptools'] + + def cythonize_build(self, env, build_dir='.'): + super(KivyRecipe, self).cythonize_build(env, build_dir=build_dir) + + if not exists(join(build_dir, 'kivy', 'include')): + return + + # If kivy is new enough to use the include dir, copy it + # manually to the right location as we bypass this stage of + # the build + with current_directory(build_dir): + build_libs_dirs = glob.glob(join('build', 'lib.*')) + + for dirn in build_libs_dirs: + shprint(sh.cp, '-r', join('kivy', 'include'), + join(dirn, 'kivy')) + + def cythonize_file(self, env, build_dir, filename): + # We can ignore a few files that aren't important to the + # android build, and may not work on Android anyway + do_not_cythonize = ['window_x11.pyx', ] + if basename(filename) in do_not_cythonize: + return + super(KivyRecipe, self).cythonize_file(env, build_dir, filename) + + def get_recipe_env(self, arch): + env = super(KivyRecipe, self).get_recipe_env(arch) + if 'sdl2' in self.ctx.recipe_build_order: + env['USE_SDL2'] = '1' + env['KIVY_SPLIT_EXAMPLES'] = '1' + env['KIVY_SDL2_PATH'] = ':'.join([ + join(self.ctx.bootstrap.build_dir, 'jni', 'SDL', 'include'), + join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_image'), + join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_mixer'), + join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_ttf'), + ]) + + return env + + +class CustomKivyRecipe(KivyRecipe): + """ + Overrides `KivyRecipe`, patches performance issue, refs: + - https://github.com/kivy/kivy/pull/6242 + - https://github.com/kivy/python-for-android/issues/2002 + """ + patches = ('revert4219.patch',) + + +recipe = CustomKivyRecipe() diff --git a/src/python-for-android/recipes/kivy/revert4219.patch b/src/python-for-android/recipes/kivy/revert4219.patch new file mode 100644 index 0000000..ec85b20 --- /dev/null +++ b/src/python-for-android/recipes/kivy/revert4219.patch @@ -0,0 +1,37 @@ +diff --git a/kivy/core/window/_window_sdl2.pyx b/kivy/core/window/_window_sdl2.pyx +index 69e82ed47..c6b056140 100644 +--- a/kivy/core/window/_window_sdl2.pyx ++++ b/kivy/core/window/_window_sdl2.pyx +@@ -6,7 +6,6 @@ from os import environ + from kivy.config import Config + from kivy.logger import Logger + from kivy import platform +-from kivy.graphics.cgl cimport * + + from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free + +@@ -652,10 +651,7 @@ cdef class _WindowSDL2Storage: + pass + + def flip(self): +- win = self.win +- with nogil: +- SDL_GL_SwapWindow(win) +- cgl.glFinish() ++ SDL_GL_SwapWindow(self.win) + + def save_bytes_in_png(self, filename, data, int width, int height): + cdef SDL_Surface *surface = SDL_CreateRGBSurfaceFrom( +diff --git a/kivy/lib/sdl2.pxi b/kivy/lib/sdl2.pxi +index f750e5d3e..e6f2b5c68 100644 +--- a/kivy/lib/sdl2.pxi ++++ b/kivy/lib/sdl2.pxi +@@ -604,7 +604,7 @@ cdef extern from "SDL.h": + cdef SDL_GLContext SDL_GL_GetCurrentContext() + cdef int SDL_GL_SetSwapInterval(int interval) + cdef int SDL_GL_GetSwapInterval() +- cdef void SDL_GL_SwapWindow(SDL_Window * window) nogil ++ cdef void SDL_GL_SwapWindow(SDL_Window * window) + cdef void SDL_GL_DeleteContext(SDL_GLContext context) + + cdef int SDL_NumJoysticks() From f74eb54f935b319e54e7a8a58603538dc5cfd9ca Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Fri, 8 Nov 2019 21:51:35 +0100 Subject: [PATCH 4/4] v2019.1108 --- src/etherollapp/CHANGELOG.md | 2 +- src/etherollapp/version.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/etherollapp/CHANGELOG.md b/src/etherollapp/CHANGELOG.md index bffa2e3..bd99091 100644 --- a/src/etherollapp/CHANGELOG.md +++ b/src/etherollapp/CHANGELOG.md @@ -1,7 +1,7 @@ # Change Log -## [Unreleased] +## [v2019.1108] - Fix Kivy low FPS performances issue - Fix 403 errors on Ropsten, refs #156, #157 diff --git a/src/etherollapp/version.py b/src/etherollapp/version.py index dc04158..e3f0f49 100644 --- a/src/etherollapp/version.py +++ b/src/etherollapp/version.py @@ -1,8 +1,7 @@ -__version__ = '2019.1101' +__version__ = '2019.1108' # The `__version_code__` is used for the F-Droid auto update and should match # the `versionCode` from the `build.gradle` file located in: # `.buildozer/android/platform/build-*/dists/etheroll__*/build.gradle` -# `.buildozer/android/platform/build/dists/etheroll/` # The auto update method used is the `HTTP`, see: # https://f-droid.org/en/docs/Build_Metadata_Reference/#UpdateCheckMode -__version_code__ = 721203001 +__version_code__ = 721203008