From 31e04f80fdf791780d37dd3c5ee91b3b1e5f1d7b Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 18:37:04 +0300 Subject: [PATCH 01/35] [EN-3260] Download api 8.3.0. Fix the listeners & pxd_include --- .gitignore | 4 + build.py | 91 ++++++++++++----- dxfeed/core/listeners/listener.pyx | 4 +- dxfeed/core/pxd_include/DXErrorCodes.pxd | 12 ++- dxfeed/core/pxd_include/DXFeed.pxd | 123 ++++++++++++++++++++++- dxfeed/core/pxd_include/DXTypes.pxd | 6 +- dxfeed/core/pxd_include/EventData.pxd | 106 ++++++++++++++----- dxfeed/core/pxd_include/RecordData.pxd | 59 +++++++---- dxfeed/dxfeed-c-api | 2 +- pyproject.toml | 12 +-- 10 files changed, 336 insertions(+), 83 deletions(-) diff --git a/.gitignore b/.gitignore index a99dcc9..d45ddee 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,7 @@ _build* Pipfile* dist* +dist +build/** +dxfeed/core/**/*.c +dxfeed/tmp diff --git a/build.py b/build.py index f0aa453..d2d2751 100644 --- a/build.py +++ b/build.py @@ -1,6 +1,12 @@ +import os +import struct + from setuptools import Extension, find_packages from setuptools.dist import Distribution from pathlib import Path +from io import BytesIO +from zipfile import ZipFile +from urllib.request import urlopen import platform try: @@ -12,49 +18,88 @@ use_cython = True ext = 'pyx' -# Get all dxfeed c api c files to be compiled into separate lib root_path = Path(__file__).resolve().parent -source_files_directory = root_path / 'dxfeed' / 'dxfeed-c-api' / 'src' -source_files_paths = [str(path) for path in source_files_directory.glob('*.c')] -libs = list() -include_dirs = [str(source_files_directory.parent / 'include'), - str(source_files_directory)] -# Build dxfeed c library -dxfeed_c_lib_args = dict() +print(root_path) + +capi_version = '8.3.0' + +is_x64 = 8 * struct.calcsize("P") == 64 + if platform.system() == 'Windows': - source_files_paths.remove(str(source_files_directory / 'Linux.c')) - libs.append('ws2_32') + current_os = 'windows' +elif platform.system() == 'Darwin': + current_os = 'macosx' else: - source_files_paths.remove(str(source_files_directory / 'Win32.c')) -dxfeed_c_lib_args.update({'sources': source_files_paths, - 'include_dirs': include_dirs}) + current_os = 'centos' + +path_to_extract = root_path / 'dxfeed' / 'tmp' +capi_root_dir = path_to_extract / f'dxfeed-c-api-{capi_version}-no-tls' -if platform.system() == 'Darwin': - dxfeed_c_lib_args.update({'macros': [('MACOSX', 'TRUE')]}) +if (not os.path.exists(path_to_extract)) or (not os.path.exists(capi_root_dir)): + url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{capi_version}/dxfeed-c-api-{capi_version}-{current_os}-no-tls.zip' + print(f'Downloading the "{url}"') + resp = urlopen(url) + zipfile = ZipFile(BytesIO(resp.read())) + print(f'Extracting to "{path_to_extract}"') + zipfile.extractall(path_to_extract) -dxfeed_c = ('dxfeed_c', dxfeed_c_lib_args) +if current_os == 'windows': + if is_x64: + capi_library_dir = str(capi_root_dir / 'bin' / 'x64') + capi_library_name = 'DXFeed_64' + else: + capi_library_dir = capi_root_dir / 'bin' / 'x86' + capi_library_name = 'DXFeed' +elif current_os == 'macosx': + if is_x64: + capi_root_dir = capi_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls' + capi_library_dir = capi_root_dir / 'bin' / 'x64' + capi_library_name = 'DXFeed_64' + else: + raise Exception('Unsupported platform') +else: + if is_x64: + capi_root_dir = capi_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls' + capi_library_dir = capi_root_dir / 'bin' / 'x64' + capi_library_name = 'DXFeed_64' + else: + raise Exception('Unsupported platform') + +if current_os == 'windows': + runtime_library_dirs = [] +else: + runtime_library_dirs = [str(capi_library_dir)] + +capi_include_dirs = [str(capi_root_dir / 'include'), str(capi_root_dir / 'src')] + +libs = [capi_library_name] +if platform.system() == 'Windows': + libs.append('ws2_32') extensions = [Extension('dxfeed.core.utils.helpers', ['dxfeed/core/utils/helpers.' + ext], - include_dirs=include_dirs), + include_dirs=capi_include_dirs), Extension('dxfeed.core.utils.handler', ['dxfeed/core/utils/handler.' + ext], - include_dirs=include_dirs), + include_dirs=capi_include_dirs), Extension('dxfeed.core.listeners.listener', ['dxfeed/core/listeners/listener.' + ext], - include_dirs=include_dirs), - Extension('dxfeed.core.DXFeedPy', ['dxfeed/core/DXFeedPy.' + ext], libraries=libs, - include_dirs=include_dirs)] + include_dirs=capi_include_dirs), + Extension('dxfeed.core.DXFeedPy', ['dxfeed/core/DXFeedPy.' + ext], library_dirs=[str(capi_library_dir)], + runtime_library_dirs=runtime_library_dirs, + libraries=libs, + include_dirs=capi_include_dirs)] if use_cython: extensions = cythonize(extensions, language_level=3) + def build(setup_kwargs): setup_kwargs.update({ 'ext_modules': extensions, 'zip_safe': False, - 'libraries': [dxfeed_c], 'packages': find_packages(), - 'include_dirs': include_dirs + 'include_dirs': capi_include_dirs }) + def build_extensions(): """ Function for building extensions inplace for docs and tests diff --git a/dxfeed/core/listeners/listener.pyx b/dxfeed/core/listeners/listener.pyx index 59ea437..76b6a7f 100644 --- a/dxfeed/core/listeners/listener.pyx +++ b/dxfeed/core/listeners/listener.pyx @@ -133,8 +133,8 @@ cdef void profile_default_listener(int event_type, div_freq=p[i].div_freq, exd_div_amount=p[i].exd_div_amount, exd_div_date=p[i].exd_div_date, - high_price=p[i]._52_high_price, - low_price=p[i]._52_low_price, + high_price=p[i].high_52_week_price, + low_price=p[i].low_52_week_price, shares=p[i].shares, free_float=p[i].free_float, high_limit_price=p[i].high_limit_price, diff --git a/dxfeed/core/pxd_include/DXErrorCodes.pxd b/dxfeed/core/pxd_include/DXErrorCodes.pxd index 35e682a..3f4b1c6 100644 --- a/dxfeed/core/pxd_include/DXErrorCodes.pxd +++ b/dxfeed/core/pxd_include/DXErrorCodes.pxd @@ -1,6 +1,14 @@ from dxfeed.core.pxd_include.DXTypes cimport * -cdef extern from "DXErrorCodes.h": +cdef extern from "": + ctypedef enum dx_log_level_t: + dx_ll_trace = -2, + dx_ll_debug = -1, + dx_ll_info = 0, + dx_ll_warn = 1, + dx_ll_error = 2 + +cdef extern from "": ctypedef enum dx_error_code_t: # # /* common error codes */ @@ -176,4 +184,6 @@ cdef extern from "DXErrorCodes.h": # # /* -------------------------------------------------------------------------- */ cdef dxf_const_string_t dx_get_error_description (dx_error_code_t code) + +cdef dx_log_level_t dx_get_log_level(dx_error_code_t code) \ No newline at end of file diff --git a/dxfeed/core/pxd_include/DXFeed.pxd b/dxfeed/core/pxd_include/DXFeed.pxd index 8d43287..02bbad6 100644 --- a/dxfeed/core/pxd_include/DXFeed.pxd +++ b/dxfeed/core/pxd_include/DXFeed.pxd @@ -8,7 +8,7 @@ from dxfeed.core.pxd_include.EventData cimport * # /* -------------------------------------------------------------------------- */ -cdef extern from "DXFeed.h": +cdef extern from "": cdef int DXF_SUCCESS = 1 cdef int DXF_FAILURE = 0 @@ -33,6 +33,10 @@ cdef extern from "DXFeed.h": dxf_connection_status_t new_status, void* user_data) + ctypedef void (*dxf_conn_on_server_heartbeat_notifier_t) (dxf_connection_t connection, dxf_long_t server_millis, + dxf_int_t server_lag_mark, dxf_int_t connection_rtt, + void * user_data); + # /* the low level callback types, required in case some thread-specific initialization must be performed # on the client side on the thread creation/destruction */ @@ -165,6 +169,24 @@ cdef extern from "DXFeed.h": void* user_data, dxf_connection_t* connection) +#/** +# * @ingroup c-api-connection-functions +# * +# * @brief Sets a server heartbeat notifier's callback to the connection. +# * +# * @details This notifier will be invoked when the new heartbeat arrives from a server and contains non empty payload +# * +# * @param[in] connection The handle of a previously created connection +# * @param[in] notifier The notifier callback function pointer +# * @param[in] user_data The data to be passed to the callback function +# * +# * @return {@link DXF_SUCCESS} on successful set of the heartbeat notifier's or {@link DXF_FAILURE} on error; +# * {@link dxf_get_last_error} can be used to retrieve the error code and description in case of failure; +# */ + ERRORCODE dxf_set_on_server_heartbeat_notifier(dxf_connection_t connection, + dxf_conn_on_server_heartbeat_notifier_t notifier, + void* user_data) + # /* # * Closes a connection. # @@ -183,6 +205,28 @@ cdef extern from "DXFeed.h": ERRORCODE dxf_create_subscription (dxf_connection_t connection, int event_types, dxf_subscription_t* subscription) + +#/** +# * @ingroup c-api-basic-subscription-functions +# * +# * @brief Creates a subscription with the specified parameters and the subscription flags. +# * +# * @details +# * +# * @param[in] connection A handle of a previously created connection which the subscription will be using +# * @param[in] event_types A bitmask of the subscription event types. See {@link dx_event_id_t} and +# * {@link DX_EVENT_BIT_MASK} for information on how to create an event type bitmask +# * @param[in] subscr_flags A bitmask of the subscription event flags. See {@link dx_event_subscr_flag} +# * @param[out] subscription A handle of the created subscription +# * +# * @return {@link DXF_SUCCESS} on successful subscription creation or {@link DXF_FAILURE} on error; +# * {@link dxf_get_last_error} can be used to retrieve the error code and description in case of failure; +# * a handle to newly created subscription is returned via ```subscription``` out parameter +# */ + ERRORCODE dxf_create_subscription_with_flags(dxf_connection_t connection, int event_types, + dx_event_subscr_flag subscr_flags, + dxf_subscription_t* subscription) + # /* # * Creates a subscription with the specified parameters. # @@ -195,6 +239,27 @@ cdef extern from "DXFeed.h": ERRORCODE dxf_create_subscription_timed(dxf_connection_t connection, int event_types, dxf_long_t time, dxf_subscription_t* subscription) +#/** +# * @ingroup c-api-basic-subscription-functions +# * +# * @brief Creates a timed subscription with the specified parameters and the subscription flags. +# * +# * @details +# * +# * @param[in] connection A handle of a previously created connection which the subscription will be using +# * @param[in] event_types A bitmask of the subscription event types. See {@link dx_event_id_t} and +# * {@link DX_EVENT_BIT_MASK} for information on how to create an event type bitmask +# * @param[in] time UTC time in the past (unix time in milliseconds) +# * @param[in] subscr_flags A bitmask of the subscription event flags. See {@link dx_event_subscr_flag} +# * @param[out] subscription A handle of the created subscription +# * +# * @return {@link DXF_SUCCESS} on successful subscription creation or {@link DXF_FAILURE} on error; +# * {@link dxf_get_last_error} can be used to retrieve the error code and description in case of failure; +# * a handle to newly created subscription is returned via ```subscription``` out parameter +# */ + ERRORCODE dxf_create_subscription_timed_with_flags(dxf_connection_t connection, int event_types, + dxf_long_t time, dx_event_subscr_flag subscr_flags, + dxf_subscription_t* subscription) # /* # * Closes a subscription. @@ -375,6 +440,30 @@ cdef extern from "DXFeed.h": # */ ERRORCODE dxf_initialize_logger (const char* file_name, int rewrite_file, int show_timezone_info, int verbose) + +#/** +# * @ingroup c-api-common +# * +# * @brief Initializes the internal logger with data transfer logging. +# * +# * @details Various actions and events, including the errors, are being logged +# * throughout the framework. They may be stored into the file. +# * +# * @param[in] file_name A full path to the file where the log is to be stored +# * @param[in] rewrite_file A flag defining the file open mode; if it's nonzero then the log file will be rewritten +# * @param[in] show_timezone_info A flag defining the time display option in the log file; if it's nonzero then +# * the time will be displayed with the timezone suffix +# * @param[in] verbose A flag defining the logging mode; if it's nonzero then the verbose logging will be +# * enabled +# * @param[in] log_data_transfer A flag defining the logging mode; if it's nonzero then the data transfer (portions of +# * received and sent data) logging will be enabled +# * +# * @return {@link DXF_SUCCESS} on successful logger initialization or {@link DXF_FAILURE} on error; +# * {@link dxf_get_last_error} can be used to retrieve the error code and description in case of failure; +# */ + ERRORCODE dxf_initialize_logger_v2(const char * file_name, int rewrite_file, int show_timezone_info, + int verbose, int log_data_transfer) + # /* # * Clear current sources and add new one to subscription # * Warning: you must configure order source before dxf_add_symbols/dxf_add_symbol call @@ -554,6 +643,28 @@ cdef extern from "DXFeed.h": dxf_const_string_t symbol, const char** sources, dxf_price_level_book_t* book) +#/** +# * @ingroup c-api-price-level-book +# * +# * @brief Creates Price Level book with the specified parameters. +# * +# * @details +# * +# * @param[in] connection A handle of a previously created connection which the subscription will be using +# * @param[in] symbol The symbol to use +# * @param[in] sources Order sources for Order. Each element can be one of following: +# * "NTV", "ntv", "NFX", "ESPD", "XNFI", "ICE", "ISE", "DEA", "DEX", "BYX", "BZX", "BATE", +# * "CHIX", "CEUX", "BXTR", "IST", "BI20", "ABE", "FAIR", "GLBX", "glbx", "ERIS", "XEUR", "xeur", "CFE", "C2OX", "SMFE", +# * "smfe", "iex", "MEMX", "memx". NULL-list means "all sources" +# * @param[in] sources_count The number of sources. 0 means "all sources" +# * @param[out] book A handle of the created price level book +# * +# * @return {@link DXF_SUCCESS} if price level book has been successfully created or {@link DXF_FAILURE} on error; +# * {@link dxf_get_last_error} can be used to retrieve the error code and description in case of failure; +# * *price level book* itself is returned via out parameter +# */ + ERRORCODE dxf_create_price_level_book_v2(dxf_connection_t connection, dxf_const_string_t symbol, + const char** sources, int sources_count, dxf_price_level_book_t* book) # /* # * Closes a price level book. # * All the data associated with it will be freed. @@ -701,4 +812,14 @@ cdef extern from "DXFeed.h": # */ ERRORCODE dxf_free(void* pointer) + dxf_const_string_t dxf_get_order_action_wstring_name(dxf_order_action_t action); + + const char * dxf_get_order_action_string_name(dxf_order_action_t action); + + ERRORCODE dxf_load_config_from_wstring(dxf_const_string_t config); + + ERRORCODE dxf_load_config_from_string(const char * config); + + ERRORCODE dxf_load_config_from_file(const char * file_name); + #endif /* _H_INCLUDED */ diff --git a/dxfeed/core/pxd_include/DXTypes.pxd b/dxfeed/core/pxd_include/DXTypes.pxd index 1d53270..c9edff4 100644 --- a/dxfeed/core/pxd_include/DXTypes.pxd +++ b/dxfeed/core/pxd_include/DXTypes.pxd @@ -3,7 +3,7 @@ #ifndef DX_TYPES_H_INCLUDED #define DX_TYPES_H_INCLUDED -cdef extern from "DXTypes.h": +cdef extern from "": ctypedef int ERRORCODE ctypedef void* dxf_subscription_t ctypedef void* dxf_connection_t @@ -17,7 +17,7 @@ cdef extern from "DXTypes.h": #pxd_include from libc.stddef cimport wchar_t -cdef extern from "DXTypes.h": +cdef extern from "": ctypedef unsigned char dxf_bool_t # 8 bit ctypedef char dxf_byte_t # 8 bit ctypedef unsigned char dxf_ubyte_t # 8 bit @@ -61,7 +61,7 @@ cdef extern from "DXTypes.h": #endif /* _WIN32/POSIX */ -cdef extern from "DXTypes.h": +cdef extern from "": ctypedef dxf_uint_t dxf_event_flags_t ctypedef struct dxf_byte_array_t: diff --git a/dxfeed/core/pxd_include/EventData.pxd b/dxfeed/core/pxd_include/EventData.pxd index 9724e48..b10d413 100644 --- a/dxfeed/core/pxd_include/EventData.pxd +++ b/dxfeed/core/pxd_include/EventData.pxd @@ -7,12 +7,26 @@ from dxfeed.core.pxd_include cimport DXTypes as dxt #define OUT #endif /* OUT */ +cdef extern from "": + enum dxf_order_action_t: + dxf_oa_undefined = 0, + dxf_oa_new = 1, + dxf_oa_replace = 2, + dxf_oa_modify = 3, + dxf_oa_delete = 4, + dxf_oa_partial = 5, + dxf_oa_execute = 6, + dxf_oa_trade = 7, + dxf_oa_bust = 8, + dxf_oa_last = dxf_oa_bust + + # /* -------------------------------------------------------------------------- */ # /* # * Event type constants # */ # /* -------------------------------------------------------------------------- */ -cdef extern from "EventData.h": +cdef extern from "": enum dx_event_id_t: dx_eid_begin = 0, dx_eid_trade = dx_eid_begin, @@ -57,14 +71,38 @@ cdef extern from "EventData.h": # // The length of record suffix including including the terminating null character #define DXF_RECORD_SUFFIX_SIZE 5 DEF DXF_RECORD_SUFFIX_SIZE = 5 -cdef extern from "EventData.h": +cdef extern from "": cdef int DXF_RECORD_SUFFIX_SIZE = DXF_RECORD_SUFFIX_SIZE + +cdef extern from "": + ctypedef enum dx_event_subscr_flag_t: +# /// (0x0) Used for default subscription + dx_esf_default = 0x0u, +# /// (0x1) Used for subscribing on one record only in case of snapshots + dx_esf_single_record = 0x1u, +# /// (0x2) Used with #dx_esf_single_record flag and for #dx_eid_order (Order) event + dx_esf_sr_market_maker_order = 0x2u, +# /// (0x4) Used for time series subscription + dx_esf_time_series = 0x4u, +# /// (0x8) Used for regional quotes + dx_esf_quotes_regional = 0x8u, +# /// (0x10) Used for wildcard ("*") subscription + dx_esf_wildcard = 0x10u, +# /// (0x20) Used for forcing subscription to ticker data + dx_esf_force_ticker = 0x20u, +# /// (0x40) Used for forcing subscription to stream data + dx_esf_force_stream = 0x40u, +# /// (0x80) Used for forcing subscription to history data + dx_esf_force_history = 0x80u + + ctypedef dx_event_subscr_flag_t dx_event_subscr_flag + # /* -------------------------------------------------------------------------- */ # /* # * Source suffix array # */ # /* -------------------------------------------------------------------------- */ -cdef extern from "EventData.h": +cdef extern from "": struct dx_suffix_t: dxt.dxf_char_t suffix[DXF_RECORD_SUFFIX_SIZE] @@ -109,18 +147,19 @@ cdef extern from "EventData.h": dxt.dxf_int_t time_nanos, dxt.dxf_char_t exchange_code, dxt.dxf_double_t price, - dxt.dxf_int_t size, + dxt.dxf_double_t size, # /* This field is absent in TradeETH */ dxt.dxf_int_t tick, - # /* This field is absent in TradeETH */ dxt.dxf_double_t change, - dxt.dxf_int_t raw_flags, + dxt.dxf_dayid_t day_id, dxt.dxf_double_t day_volume, dxt.dxf_double_t day_turnover, + dxt.dxf_int_t raw_flags, dxf_direction_t direction, dxt.dxf_bool_t is_eth, dxf_order_scope_t scope + ctypedef dxf_trade_t dxf_trade_eth_t # /* Quote -------------------------------------------------------------------- */ @@ -131,11 +170,11 @@ cdef extern from "EventData.h": dxt.dxf_long_t bid_time, dxt.dxf_char_t bid_exchange_code, dxt.dxf_double_t bid_price, - dxt.dxf_int_t bid_size, + dxt.dxf_double_t bid_size, dxt.dxf_long_t ask_time, dxt.dxf_char_t ask_exchange_code, dxt.dxf_double_t ask_price, - dxt.dxf_int_t ask_size, + dxt.dxf_double_t ask_size, dxf_order_scope_t scope @@ -157,7 +196,7 @@ cdef extern from "EventData.h": dxt.dxf_dayid_t prev_day_id, dxt.dxf_double_t prev_day_close_price, dxt.dxf_double_t prev_day_volume, - dxt.dxf_int_t open_interest, + dxt.dxf_double_t open_interest, dxt.dxf_int_t raw_flags, dxt.dxf_char_t exchange_code, dxf_price_type_t day_close_price_type, @@ -182,11 +221,11 @@ cdef extern from "EventData.h": ctypedef struct dxf_profile_t: dxt.dxf_double_t beta, dxt.dxf_double_t eps, - dxt.dxf_int_t div_freq, + dxt.dxf_double_t div_freq, dxt.dxf_double_t exd_div_amount, dxt.dxf_dayid_t exd_div_date, - dxt.dxf_double_t _52_high_price, - dxt.dxf_double_t _52_low_price, + dxt.dxf_double_t high_52_week_price, + dxt.dxf_double_t low_52_week_price, dxt.dxf_double_t shares, dxt.dxf_double_t free_float, dxt.dxf_double_t high_limit_price, @@ -213,18 +252,26 @@ cdef extern from "EventData.h": ctypedef struct dxf_order_t: + dxt.dxf_char_t source[DXF_RECORD_SUFFIX_SIZE], dxt.dxf_event_flags_t event_flags, dxt.dxf_long_t index, dxt.dxf_long_t time, - dxt.dxf_int_t time_nanos, dxt.dxf_int_t sequence, + dxt.dxf_int_t time_nanos, + dxf_order_action_t action, + dxt.dxf_long_t action_time, + dxt.dxf_long_t order_id, + dxt.dxf_long_t aux_order_id, dxt.dxf_double_t price, - dxt.dxf_int_t size, - dxt.dxf_int_t count, - dxf_order_scope_t scope, - dxf_order_side_t side, + dxt.dxf_double_t size, + dxt.dxf_double_t executed_size, + dxt.dxf_double_t count, + dxt.dxf_long_t trade_id, + dxt.dxf_double_t trade_price, + dxt.dxf_double_t trade_size, dxt.dxf_char_t exchange_code, - dxt.dxf_char_t source[DXF_RECORD_SUFFIX_SIZE], + dxf_order_side_t side, + dxf_order_scope_t scope, # _inner_oso # probably there should be some name (p58 of cython book) dxt.dxf_const_string_t market_maker, dxt.dxf_const_string_t spread_symbol @@ -243,7 +290,7 @@ cdef extern from "EventData.h": dxt.dxf_long_t time, dxt.dxf_char_t exchange_code, dxt.dxf_double_t price, - dxt.dxf_int_t size, + dxt.dxf_double_t size, dxt.dxf_double_t bid_price, dxt.dxf_double_t ask_price, dxt.dxf_const_string_t exchange_sale_conditions, @@ -274,7 +321,7 @@ cdef extern from "EventData.h": dxt.dxf_double_t vwap, dxt.dxf_double_t bid_volume, dxt.dxf_double_t ask_volume, - dxt.dxf_int_t open_interest, + dxt.dxf_double_t open_interest, dxt.dxf_double_t imp_volatility @@ -296,8 +343,16 @@ cdef extern from "EventData.h": ctypedef rd.dx_theo_price_t dxf_theo_price_t # /* Underlying --------------------------------------------------------------- */ -# /* Event and record are the same */ - ctypedef rd.dx_underlying_t dxf_underlying_t + ctypedef struct dxf_underlying_t: + dxt.dxf_double_t volatility, + dxt.dxf_double_t front_volatility, + dxt.dxf_double_t back_volatility, + dxt.dxf_double_t call_volume, + dxt.dxf_double_t put_volume, + dxt.dxf_double_t option_volume, + dxt.dxf_double_t put_call_ratio + + ctypedef dxf_underlying_t dxf_underlying # /* Series ------------------------------------------------------------------- */ ctypedef struct dxf_series_t: @@ -307,6 +362,9 @@ cdef extern from "EventData.h": dxt.dxf_int_t sequence, dxt.dxf_dayid_t expiration, dxt.dxf_double_t volatility, + dxt.dxf_double_t call_volume, + dxt.dxf_double_t put_volume, + dxt.dxf_double_t option_volume, dxt.dxf_double_t put_call_ratio, dxt.dxf_double_t forward_price, dxt.dxf_double_t dividend, @@ -445,7 +503,7 @@ cdef extern from "EventData.h": # * Various event functions # */ # /* -------------------------------------------------------------------------- */ -cdef extern from "EventData.h": +cdef extern from "": cdef dxt.dxf_const_string_t dx_event_type_to_string (int event_type) cdef int dx_get_event_data_struct_size (int event_id) cdef dx_event_id_t dx_get_event_id_by_bitmask (int event_bitmask) @@ -538,7 +596,7 @@ cdef extern from "EventData.h": # /* -------------------------------------------------------------------------- */ ctypedef struct dxf_price_level_element_t: dxt.dxf_double_t price - dxt.dxf_long_t size + dxt.dxf_double_t size dxt.dxf_long_t time diff --git a/dxfeed/core/pxd_include/RecordData.pxd b/dxfeed/core/pxd_include/RecordData.pxd index f096ad5..b47cda8 100644 --- a/dxfeed/core/pxd_include/RecordData.pxd +++ b/dxfeed/core/pxd_include/RecordData.pxd @@ -31,7 +31,7 @@ from dxfeed.core.pxd_include.DXTypes cimport * # * Record type constants # */ # /* -------------------------------------------------------------------------- */ -cdef extern from "RecordData.h": +cdef extern from "": ctypedef enum dx_record_info_id_t: dx_rid_begin = 0, dx_rid_trade = dx_rid_begin, @@ -74,12 +74,13 @@ cdef extern from "RecordData.h": dxf_int_t time_nanos dxf_char_t exchange_code dxf_double_t price - dxf_int_t size + dxf_double_t size dxf_int_t tick dxf_double_t change - dxf_int_t flags + dxf_dayid_t day_id; dxf_double_t day_volume dxf_double_t day_turnover + dxf_int_t flags ctypedef dx_trade_t dx_trade_eth_t @@ -89,11 +90,11 @@ cdef extern from "RecordData.h": dxf_int_t bid_time dxf_char_t bid_exchange_code dxf_double_t bid_price - dxf_int_t bid_size + dxf_double_t bid_size dxf_int_t ask_time dxf_char_t ask_exchange_code dxf_double_t ask_price - dxf_int_t ask_size + dxf_double_t ask_size ctypedef struct dx_summary_t: dxf_dayid_t day_id @@ -104,18 +105,18 @@ cdef extern from "RecordData.h": dxf_dayid_t prev_day_id dxf_double_t prev_day_close_price dxf_double_t prev_day_volume - dxf_int_t open_interest + dxf_double_t open_interest dxf_int_t flags ctypedef struct dx_profile_t: dxf_double_t beta dxf_double_t eps - dxf_int_t div_freq + dxf_double_t div_freq dxf_double_t exd_div_amount dxf_dayid_t exd_div_date - dxf_double_t _52_high_price - dxf_double_t _52_low_price + dxf_double_t high_price_52 + dxf_double_t low_price_52 dxf_double_t shares dxf_double_t free_float dxf_double_t high_limit_price @@ -126,28 +127,35 @@ cdef extern from "RecordData.h": dxf_const_string_t description dxf_const_string_t status_reason -cdef extern from "RecordData.h": +cdef extern from "": ctypedef struct dx_market_maker_t: dxf_char_t mm_exchange dxf_int_t mm_id dxf_int_t mmbid_time dxf_double_t mmbid_price - dxf_int_t mmbid_size - dxf_int_t mmbid_count + dxf_double_t mmbid_size + dxf_double_t mmbid_count dxf_int_t mmask_time dxf_double_t mmask_price - dxf_int_t mmask_size - dxf_int_t mmask_count + dxf_double_t mmask_size + dxf_double_t mmask_count ctypedef struct dx_order_t: dxf_int_t index dxf_int_t time dxf_int_t time_nanos dxf_int_t sequence + dxf_long_t action_time; + dxf_long_t order_id; + dxf_long_t aux_order_id; dxf_double_t price - dxf_int_t size - dxf_int_t count + dxf_double_t size + dxf_double_t executed_size; + dxf_double_t count dxf_int_t flags + dxf_long_t trade_id; + dxf_double_t trade_price; + dxf_double_t trade_size; dxf_int_t mmid @@ -156,10 +164,17 @@ cdef extern from "RecordData.h": dxf_int_t time dxf_int_t time_nanos dxf_int_t sequence + dxf_long_t action_time; + dxf_long_t order_id; + dxf_long_t aux_order_id; dxf_double_t price - dxf_int_t size - dxf_int_t count + dxf_double_t size + dxf_double_t executed_size; + dxf_double_t count dxf_int_t flags + dxf_long_t trade_id; + dxf_double_t trade_price; + dxf_double_t trade_size; dxf_const_string_t spread_symbol @@ -168,7 +183,7 @@ cdef extern from "RecordData.h": dxf_int_t sequence dxf_char_t exchange_code dxf_double_t price - dxf_int_t size + dxf_double_t size dxf_double_t bid_price dxf_double_t ask_price dxf_int_t exchange_sale_conditions @@ -188,7 +203,7 @@ cdef extern from "RecordData.h": dxf_double_t vwap dxf_double_t bid_volume dxf_double_t ask_volume - dxf_int_t open_interest + dxf_double_t open_interest dxf_double_t imp_volatility ctypedef struct dx_greeks_t: @@ -216,6 +231,8 @@ cdef extern from "RecordData.h": dxf_double_t volatility dxf_double_t front_volatility dxf_double_t back_volatility + dxf_double_t call_volume; + dxf_double_t put_volume; dxf_double_t put_call_ratio ctypedef struct dx_series_t: @@ -224,6 +241,8 @@ cdef extern from "RecordData.h": dxf_int_t sequence dxf_dayid_t expiration dxf_double_t volatility + dxf_double_t call_volume; + dxf_double_t put_volume; dxf_double_t put_call_ratio dxf_double_t forward_price dxf_double_t dividend diff --git a/dxfeed/dxfeed-c-api b/dxfeed/dxfeed-c-api index d3c05fd..8b066ec 160000 --- a/dxfeed/dxfeed-c-api +++ b/dxfeed/dxfeed-c-api @@ -1 +1 @@ -Subproject commit d3c05fdb5c982bb62946a9c373bcbb1bc0bb44d0 +Subproject commit 8b066ec42fc3ebef9c0816ece6b6eb7a6fa97099 diff --git a/pyproject.toml b/pyproject.toml index c954472..b0f0e13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,13 +16,9 @@ classifiers = ['Development Status :: 4 - Beta', 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', 'Natural Language :: English'] -include = ['LICENSE', 'dxfeed/**/*.c'] -exclude = ['dxfeed/dxfeed-c-api/wrappers*', - 'dxfeed/dxfeed-c-api/tests*', - 'dxfeed/dxfeed-c-api/scripts*', - 'dxfeed/dxfeed-c-api/lib*', - 'dxfeed/dxfeed-c-api/docs*', - 'dxfeed/dxfeed-c-api/.github*'] +include = ['LICENSE'] +#include = ['LICENSE', 'dxfeed/**/*.c'] +exclude = ['dxfeed/dxfeed-c-api*', 'dxfeed/tmp*'] [tool.poetry.urls] 'Source code' = 'https://github.com/dxFeed/dxfeed-python-api' @@ -30,7 +26,7 @@ exclude = ['dxfeed/dxfeed-c-api/wrappers*', [tool.poetry.dependencies] python = '^3.6' -pandas = '^0.25.1' +pandas = '^1.0.0' toml = '^0.10.0' # The following dependencies are used for docs generation when installed as extras # (e.g. pip install nornir[docs]) From 20693f138859db5408541304e3478246b106af8c Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 18:45:40 +0300 Subject: [PATCH 02/35] [EN-3260] remove submodule --- .gitmodules | 3 --- dxfeed/dxfeed-c-api | 1 - 2 files changed, 4 deletions(-) delete mode 100644 .gitmodules delete mode 160000 dxfeed/dxfeed-c-api diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index c34da26..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "dxfeed/dxfeed-c-api"] - path = dxfeed/dxfeed-c-api - url = https://github.com/dxFeed/dxfeed-c-api.git diff --git a/dxfeed/dxfeed-c-api b/dxfeed/dxfeed-c-api deleted file mode 160000 index 8b066ec..0000000 --- a/dxfeed/dxfeed-c-api +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8b066ec42fc3ebef9c0816ece6b6eb7a6fa97099 From fdc9790edda460fc3e609297c12328e0ed7293c2 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 21:16:29 +0300 Subject: [PATCH 03/35] [EN-3260] fix build, clear tests --- .gitignore | 4 +++ README.md | 6 ---- build.py | 58 +++++++++++++++++++++++--------- clear.py | 23 +++++++++++++ pyproject.toml | 23 ++++++------- tests/test_dxfeedpy.py | 6 ++-- tests/test_subscription_class.py | 8 ++--- 7 files changed, 87 insertions(+), 41 deletions(-) create mode 100644 clear.py diff --git a/.gitignore b/.gitignore index d45ddee..c8d504c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,7 @@ dist build/** dxfeed/core/**/*.c dxfeed/tmp + +setup.py +poetry.lock +dxfeed.egg-info diff --git a/README.md b/README.md index f8cb9d4..f0d0010 100644 --- a/README.md +++ b/README.md @@ -34,12 +34,6 @@ pip3 install dxfeed ## Installation from sources -Reminder: initialize and pull git submodule after cloning the repo: -```bash -git submodule init -git submodule update -``` - To install dxfeed from source you need Poetry. It provides a custom installer. This is the recommended way of installing poetry according to [documentation](https://python-poetry.org/docs/) diff --git a/build.py b/build.py index d2d2751..4d0731a 100644 --- a/build.py +++ b/build.py @@ -1,4 +1,5 @@ import os +import shutil import struct from setuptools import Extension, find_packages @@ -32,45 +33,66 @@ else: current_os = 'centos' +capi_root_dir = root_path / 'dxfeed' / 'dxfeed-c-api' +capi_include_dir = capi_root_dir / 'include' +# capi_bin_dir = capi_root_dir / 'bin' +capi_bin_dir = root_path / 'dxfeed' / 'core' path_to_extract = root_path / 'dxfeed' / 'tmp' -capi_root_dir = path_to_extract / f'dxfeed-c-api-{capi_version}-no-tls' +capi_extracted_root_dir = path_to_extract / f'dxfeed-c-api-{capi_version}-no-tls' -if (not os.path.exists(path_to_extract)) or (not os.path.exists(capi_root_dir)): - url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{capi_version}/dxfeed-c-api-{capi_version}-{current_os}-no-tls.zip' +if (not os.path.exists(path_to_extract)) or (not os.path.exists(capi_extracted_root_dir)): + url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{capi_version}/dxfeed-c-api-{capi_version}-' \ + f'{current_os}-no-tls.zip ' print(f'Downloading the "{url}"') resp = urlopen(url) zipfile = ZipFile(BytesIO(resp.read())) print(f'Extracting to "{path_to_extract}"') zipfile.extractall(path_to_extract) -if current_os == 'windows': +if platform.system() == 'Windows': if is_x64: - capi_library_dir = str(capi_root_dir / 'bin' / 'x64') + capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x64' capi_library_name = 'DXFeed_64' + capi_library_file_name = f'{capi_library_name}.dll' + capi_library_file_name2 = f'{capi_library_name}.lib' else: - capi_library_dir = capi_root_dir / 'bin' / 'x86' + capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x86' capi_library_name = 'DXFeed' -elif current_os == 'macosx': + capi_library_file_name = f'{capi_library_name}.dll' + capi_library_file_name2 = f'{capi_library_name}.lib' +elif platform.system() == 'Darwin': if is_x64: - capi_root_dir = capi_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls' - capi_library_dir = capi_root_dir / 'bin' / 'x64' + capi_extracted_root_dir = capi_extracted_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls' + capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x64' capi_library_name = 'DXFeed_64' + capi_library_file_name = f'lib{capi_library_name}.dylib' else: raise Exception('Unsupported platform') else: if is_x64: - capi_root_dir = capi_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls' - capi_library_dir = capi_root_dir / 'bin' / 'x64' + capi_extracted_root_dir = capi_extracted_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls' + capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x64' capi_library_name = 'DXFeed_64' + capi_library_file_name = f'lib{capi_library_name}.so' else: raise Exception('Unsupported platform') -if current_os == 'windows': - runtime_library_dirs = [] +if not os.path.exists(capi_include_dir): + shutil.copytree(capi_extracted_root_dir / 'include', capi_include_dir) +if not os.path.exists(capi_bin_dir / capi_library_file_name): + # os.makedirs(capi_bin_dir) + shutil.copyfile(capi_extracted_library_dir / capi_library_file_name, capi_bin_dir / capi_library_file_name) +if platform.system() == 'Windows': + # noinspection PyUnboundLocalVariable + if not os.path.exists(capi_bin_dir / capi_library_file_name2): + shutil.copyfile(capi_extracted_library_dir / capi_library_file_name2, capi_bin_dir / capi_library_file_name2) + +if platform.system() == 'Windows': + runtime_library_dirs = None else: - runtime_library_dirs = [str(capi_library_dir)] + runtime_library_dirs = [str(capi_bin_dir)] -capi_include_dirs = [str(capi_root_dir / 'include'), str(capi_root_dir / 'src')] +capi_include_dirs = [str(capi_include_dir)] libs = [capi_library_name] if platform.system() == 'Windows': @@ -82,7 +104,7 @@ include_dirs=capi_include_dirs), Extension('dxfeed.core.listeners.listener', ['dxfeed/core/listeners/listener.' + ext], include_dirs=capi_include_dirs), - Extension('dxfeed.core.DXFeedPy', ['dxfeed/core/DXFeedPy.' + ext], library_dirs=[str(capi_library_dir)], + Extension('dxfeed.core.DXFeedPy', ['dxfeed/core/DXFeedPy.' + ext], library_dirs=[str(capi_bin_dir)], runtime_library_dirs=runtime_library_dirs, libraries=libs, include_dirs=capi_include_dirs)] @@ -115,3 +137,7 @@ def build_extensions(): build_ext_cmd.ensure_finalized() build_ext_cmd.inplace = 1 build_ext_cmd.run() + + print(f'Removing the "{path_to_extract}"') + if os.path.exists(path_to_extract): + shutil.rmtree(path_to_extract) diff --git a/clear.py b/clear.py new file mode 100644 index 0000000..e467611 --- /dev/null +++ b/clear.py @@ -0,0 +1,23 @@ +import glob +import os +import shutil + +c_files = glob.glob('dxfeed/core/**/*.c', recursive=True) +cpp_files = glob.glob('dxfeed/core/**/*.cpp', recursive=True) +pyd_files = glob.glob('dxfeed/core/**/*.pyd', recursive=True) +so_files = glob.glob('dxfeed/core/**/*.so', recursive=True) +dll_files = glob.glob('dxfeed/core/**/*.dll', recursive=True) +lib_files = glob.glob('dxfeed/core/**/*.lib', recursive=True) +dylib_files = glob.glob('dxfeed/core/**/*.dylib', recursive=True) + +for file_path in c_files + cpp_files + pyd_files + so_files + dll_files + lib_files + dylib_files: + os.remove(file_path) + +if os.path.exists('dxfeed/tmp'): + shutil.rmtree('dxfeed/tmp') + +if os.path.exists('dxfeed/dxfeed-c-api/bin'): + shutil.rmtree('dxfeed/dxfeed-c-api/bin') + +if os.path.exists('dxfeed/dxfeed-c-api/include'): + shutil.rmtree('dxfeed/dxfeed-c-api/include') diff --git a/pyproject.toml b/pyproject.toml index b0f0e13..53e9f38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,8 +17,7 @@ classifiers = ['Development Status :: 4 - Beta', 'Natural Language :: English'] include = ['LICENSE'] -#include = ['LICENSE', 'dxfeed/**/*.c'] -exclude = ['dxfeed/dxfeed-c-api*', 'dxfeed/tmp*'] +exclude = ['dxfeed/tmp*'] [tool.poetry.urls] 'Source code' = 'https://github.com/dxFeed/dxfeed-python-api' @@ -26,29 +25,29 @@ exclude = ['dxfeed/dxfeed-c-api*', 'dxfeed/tmp*'] [tool.poetry.dependencies] python = '^3.6' -pandas = '^1.0.0' -toml = '^0.10.0' +pandas = [{version = '^1.0.0', python = '^3.6.1'}, {version = '^1.3.2', python = '^3.7.1'}] +toml = '^0.10.2' # The following dependencies are used for docs generation when installed as extras # (e.g. pip install nornir[docs]) # Currently they have to be specified as non-dev dependencies with optional = true # Refer to: https://github.com/sdispater/poetry/issues/129 # Issue to watch for: https://github.com/python-poetry/poetry/issues/1644 jupyter = { version = '^1.0.0', optional = true } -cython = { version = '^0.29.13', optional = true } +cython = { version = '^0.29.14', optional = true } [tool.poetry.dev-dependencies] -cython = '^0.29.13' -taskipy = '^1.1.3' -pytest = '^5.3.5' -sphinx = '^2.4.4' -sphinx_rtd_theme = '^0.4.3' -pygments = '^2.6.1' +cython = '^0.29.14' +taskipy = '^1.8.1' +pytest = '^6.2.4' +sphinx = '^4.1.2' +sphinx_rtd_theme = '^0.5.2' +pygments = '^2.10' [tool.poetry.extras] docs = ['jupyter', 'cython'] [tool.taskipy.tasks] -clear = 'find dxfeed/core -type f \( -iname *.c -o -iname *.cpp -o -iname *.pyd -o -iname *.so \) -delete' +clear = 'python clear.py' build = 'python build.py && poetry build' build_inplace = 'python -c "from build import *; build_extensions()"' html_docs = 'make html -C docs' diff --git a/tests/test_dxfeedpy.py b/tests/test_dxfeedpy.py index b8f97c3..3684226 100644 --- a/tests/test_dxfeedpy.py +++ b/tests/test_dxfeedpy.py @@ -90,7 +90,7 @@ def test_symbol_addition(connection): dxc.dxf_add_symbols(sc=sub, symbols=symbols) actual_symbols = dxc.dxf_get_symbols(sc=sub) dxc.dxf_close_subscription(sub) - assert symbols == actual_symbols + assert set(symbols) == set(actual_symbols) def test_symbol_deletion(connection): @@ -110,7 +110,7 @@ def test_wrong_symbol_types_ignored(connection): dxc.dxf_add_symbols(sc=sub, symbols=symbols + [1, 5.0, [], True, {}, ()]) actual_symbols = dxc.dxf_get_symbols(sc=sub) dxc.dxf_close_subscription(sub) - assert symbols == actual_symbols + assert set(symbols) == set(actual_symbols) def test_symbol_clearing(connection): @@ -155,4 +155,4 @@ def test_double_event_handler_attachment(connection): dxc.dxf_add_symbols(sub, symbols) sub.set_event_handler(handler2) assert sub.get_event_handler() is handler2 - assert dxc.dxf_get_symbols(sub) == symbols + assert set(dxc.dxf_get_symbols(sub)) == set(symbols) diff --git a/tests/test_subscription_class.py b/tests/test_subscription_class.py index 5b79ebf..035605c 100644 --- a/tests/test_subscription_class.py +++ b/tests/test_subscription_class.py @@ -42,24 +42,24 @@ def test_subscription_timed_event_type_property(subscription_timed): def test_subscription_event_type_property(subscription): subscription = subscription.add_symbols(ValueStorage.symbols) - assert subscription.symbols == ValueStorage.symbols + assert set(subscription.symbols) == set(ValueStorage.symbols) def test_subscription_timed_event_type_property(subscription_timed): subscription = subscription_timed.add_symbols(ValueStorage.symbols) - assert subscription.symbols == ValueStorage.symbols + assert set(subscription.symbols) == set(ValueStorage.symbols) def test_remove_symbol(subscription): subscription = subscription.add_symbols(ValueStorage.symbols) \ .remove_symbols([ValueStorage.symbols[0]]) - assert subscription.symbols == ValueStorage.symbols[1:] + assert set(subscription.symbols) == set(ValueStorage.symbols[1:]) def test_remove_symbol_timed(subscription_timed): subscription_timed = subscription_timed.add_symbols(ValueStorage.symbols) \ .remove_symbols([ValueStorage.symbols[0]]) - assert subscription_timed.symbols == ValueStorage.symbols[1:] + assert set(subscription_timed.symbols) == set(ValueStorage.symbols[1:]) def test_remove_all_symbols(subscription): From 33430cdbc59d4ae221fbaa557b7d430bf0b09f25 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 21:25:41 +0300 Subject: [PATCH 04/35] [EN-3260] fix build, clear tests --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 4d0731a..144c329 100644 --- a/build.py +++ b/build.py @@ -90,7 +90,7 @@ if platform.system() == 'Windows': runtime_library_dirs = None else: - runtime_library_dirs = [str(capi_bin_dir)] + runtime_library_dirs = ['.'] capi_include_dirs = [str(capi_include_dir)] From 06253f3a7b84fb560016ab8aec17e4584c442102 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 21:30:03 +0300 Subject: [PATCH 05/35] [EN-3260] extra compiler args --- build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 144c329..a781ebe 100644 --- a/build.py +++ b/build.py @@ -89,8 +89,10 @@ if platform.system() == 'Windows': runtime_library_dirs = None + extra_compiler_args = None else: - runtime_library_dirs = ['.'] + runtime_library_dirs = [str(capi_bin_dir)] + extra_compiler_args = ['-fPIC -Wl,-rpath=.'] capi_include_dirs = [str(capi_include_dir)] @@ -106,6 +108,7 @@ include_dirs=capi_include_dirs), Extension('dxfeed.core.DXFeedPy', ['dxfeed/core/DXFeedPy.' + ext], library_dirs=[str(capi_bin_dir)], runtime_library_dirs=runtime_library_dirs, + extra_compiler_args=extra_compiler_args, libraries=libs, include_dirs=capi_include_dirs)] From 38e159198b7512d0e46329a65de23501a54a0fe4 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 21:34:07 +0300 Subject: [PATCH 06/35] [EN-3260] extra compiler args --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index a781ebe..2e26f8c 100644 --- a/build.py +++ b/build.py @@ -92,7 +92,7 @@ extra_compiler_args = None else: runtime_library_dirs = [str(capi_bin_dir)] - extra_compiler_args = ['-fPIC -Wl,-rpath=.'] + extra_compiler_args = [f'-fPIC -Wl,-rpath={str(capi_bin_dir)}'] capi_include_dirs = [str(capi_include_dir)] From 657f1e4e620ac0779a61f2058847e9b778f8cd5d Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 21:42:41 +0300 Subject: [PATCH 07/35] [EN-3260] rpath --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 2e26f8c..1c23789 100644 --- a/build.py +++ b/build.py @@ -92,7 +92,7 @@ extra_compiler_args = None else: runtime_library_dirs = [str(capi_bin_dir)] - extra_compiler_args = [f'-fPIC -Wl,-rpath={str(capi_bin_dir)}'] + extra_compiler_args = [f'-fPIC -Wl,-rpath,{str(capi_bin_dir)}'] capi_include_dirs = [str(capi_include_dir)] From 9ee8ec39ca3ab98de0966b27b765a19bfc365d2b Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 22:12:32 +0300 Subject: [PATCH 08/35] [EN-3260] rpath --- build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 1c23789..211fb2f 100644 --- a/build.py +++ b/build.py @@ -91,8 +91,9 @@ runtime_library_dirs = None extra_compiler_args = None else: - runtime_library_dirs = [str(capi_bin_dir)] - extra_compiler_args = [f'-fPIC -Wl,-rpath,{str(capi_bin_dir)}'] + runtime_library_dirs = ["$ORIGIN"] + extra_compiler_args = None +print(extra_compiler_args) capi_include_dirs = [str(capi_include_dir)] From 5ffc9e074aab9ddf383cec47734ed736e99abe82 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 22:18:53 +0300 Subject: [PATCH 09/35] [EN-3260] runtime_library_dirs --- build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 211fb2f..feb860a 100644 --- a/build.py +++ b/build.py @@ -90,10 +90,12 @@ if platform.system() == 'Windows': runtime_library_dirs = None extra_compiler_args = None +elif platform.system() == 'Darwin': + runtime_library_dirs = ["@loader_path"] + extra_compiler_args = None else: runtime_library_dirs = ["$ORIGIN"] extra_compiler_args = None -print(extra_compiler_args) capi_include_dirs = [str(capi_include_dir)] From 158c117bae8a372adc2d749ccdafd0e64e9c9841 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 22:44:37 +0300 Subject: [PATCH 10/35] [EN-3260] runtime_library_dirs --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index feb860a..793fe66 100644 --- a/build.py +++ b/build.py @@ -91,10 +91,10 @@ runtime_library_dirs = None extra_compiler_args = None elif platform.system() == 'Darwin': - runtime_library_dirs = ["@loader_path"] + runtime_library_dirs = ['.', str(capi_bin_dir)] extra_compiler_args = None else: - runtime_library_dirs = ["$ORIGIN"] + runtime_library_dirs = ["$ORIGIN", '.', str(capi_bin_dir)] extra_compiler_args = None capi_include_dirs = [str(capi_include_dir)] From 41f7922e967fa8185bd8131954939c787d6c7416 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 23:13:00 +0300 Subject: [PATCH 11/35] [EN-3260] runtime_library_dirs --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 793fe66..e101ca2 100644 --- a/build.py +++ b/build.py @@ -92,7 +92,7 @@ extra_compiler_args = None elif platform.system() == 'Darwin': runtime_library_dirs = ['.', str(capi_bin_dir)] - extra_compiler_args = None + extra_compiler_args = [f'-Wl,-rpath,{str(capi_bin_dir)}'] else: runtime_library_dirs = ["$ORIGIN", '.', str(capi_bin_dir)] extra_compiler_args = None From 49df79ced3ecebe57bb392d0a7237baa15ab83c2 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 23:23:28 +0300 Subject: [PATCH 12/35] [EN-3260] runtime_library_dirs --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index e101ca2..e531508 100644 --- a/build.py +++ b/build.py @@ -92,7 +92,7 @@ extra_compiler_args = None elif platform.system() == 'Darwin': runtime_library_dirs = ['.', str(capi_bin_dir)] - extra_compiler_args = [f'-Wl,-rpath,{str(capi_bin_dir)}'] + extra_compiler_args = [f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] else: runtime_library_dirs = ["$ORIGIN", '.', str(capi_bin_dir)] extra_compiler_args = None From ddc871ac1749f38a76fc92b88e428df4fd3cb5be Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sat, 28 Aug 2021 23:39:39 +0300 Subject: [PATCH 13/35] [EN-3260] runtime_library_dirs + @loader_path --- build.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build.py b/build.py index e531508..7c204f2 100644 --- a/build.py +++ b/build.py @@ -89,13 +89,13 @@ if platform.system() == 'Windows': runtime_library_dirs = None - extra_compiler_args = None + extra_link_args = None elif platform.system() == 'Darwin': - runtime_library_dirs = ['.', str(capi_bin_dir)] - extra_compiler_args = [f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] + runtime_library_dirs = None + extra_link_args = [f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}', f'-Wl,-rpath,@loader_path/dxfeed/core'] else: runtime_library_dirs = ["$ORIGIN", '.', str(capi_bin_dir)] - extra_compiler_args = None + extra_link_args = None capi_include_dirs = [str(capi_include_dir)] @@ -111,7 +111,7 @@ include_dirs=capi_include_dirs), Extension('dxfeed.core.DXFeedPy', ['dxfeed/core/DXFeedPy.' + ext], library_dirs=[str(capi_bin_dir)], runtime_library_dirs=runtime_library_dirs, - extra_compiler_args=extra_compiler_args, + extra_link_args=extra_link_args, libraries=libs, include_dirs=capi_include_dirs)] From a40e186bd21574d27d4bb4856d6d35831cd3b6f5 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 00:12:35 +0300 Subject: [PATCH 14/35] [EN-3260] install_name_tool --- build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 7c204f2..beaa0f3 100644 --- a/build.py +++ b/build.py @@ -1,6 +1,7 @@ import os import shutil import struct +from subprocess import run from setuptools import Extension, find_packages from setuptools.dist import Distribution @@ -92,7 +93,9 @@ extra_link_args = None elif platform.system() == 'Darwin': runtime_library_dirs = None - extra_link_args = [f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}', f'-Wl,-rpath,@loader_path/dxfeed/core'] + extra_link_args = [f'-Wl,-rpath,@loader_path/dxfeed/core', f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] + capi_full_library_file_path = capi_bin_dir / capi_library_file_name + run(('install_name_tool', '-id', f'@rpath/{capi_library_file_name}', str(capi_full_library_file_path))) else: runtime_library_dirs = ["$ORIGIN", '.', str(capi_bin_dir)] extra_link_args = None From 1aff4e251dd3370f3439a60c530396a42487da0a Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 08:55:55 +0300 Subject: [PATCH 15/35] [EN-3260] install_name_tool --- build.py | 31 ++++++++++++++++++++++++++----- pyproject.toml | 2 +- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/build.py b/build.py index beaa0f3..9a15e5b 100644 --- a/build.py +++ b/build.py @@ -1,10 +1,12 @@ import os +import sys import shutil import struct from subprocess import run from setuptools import Extension, find_packages from setuptools.dist import Distribution +from setuptools.command.build_ext import build_ext from pathlib import Path from io import BytesIO from zipfile import ZipFile @@ -92,12 +94,12 @@ runtime_library_dirs = None extra_link_args = None elif platform.system() == 'Darwin': - runtime_library_dirs = None - extra_link_args = [f'-Wl,-rpath,@loader_path/dxfeed/core', f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] - capi_full_library_file_path = capi_bin_dir / capi_library_file_name - run(('install_name_tool', '-id', f'@rpath/{capi_library_file_name}', str(capi_full_library_file_path))) + runtime_library_dirs = ['.', str(capi_bin_dir)] + extra_link_args = None # [f'-Wl,-rpath,@loader_path/dxfeed/core', f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] + # capi_full_library_file_path = capi_bin_dir / capi_library_file_name + # run(('install_name_tool', '-id', f'@rpath/{capi_library_file_name}', str(capi_full_library_file_path))) else: - runtime_library_dirs = ["$ORIGIN", '.', str(capi_bin_dir)] + runtime_library_dirs = ['$ORIGIN', '.', str(capi_bin_dir)] extra_link_args = None capi_include_dirs = [str(capi_include_dir)] @@ -122,8 +124,27 @@ extensions = cythonize(extensions, language_level=3) +class custom_build_ext(build_ext): + """ + Add appropriate rpath linker flags (necessary on mac) + """ + + def finalize_options(self): + super().finalize_options() + # Special treatment of rpath in case of OSX, to work around python + # distutils bug 36353. This constructs proper rpath arguments for clang. + # See https://bugs.python.org/issue36353 + # Workaround from https://github.com/python/cpython/pull/12418 + if sys.platform[:6] == "darwin": + for path in self.rpath: + for ext in self.extensions: + ext.extra_link_args.append("-Wl,-rpath," + path) + self.rpath[:] = [] + + def build(setup_kwargs): setup_kwargs.update({ + 'cmdclass': {'build_ext': custom_build_ext}, 'ext_modules': extensions, 'zip_safe': False, 'packages': find_packages(), diff --git a/pyproject.toml b/pyproject.toml index 53e9f38..b4f6f1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,5 +56,5 @@ post_build = 'task clear' post_test = 'task clear' [build-system] -requires = ['poetry_core>=1.0.0', 'setuptools', 'cython'] +requires = ['poetry_core>=1.0.0', 'setuptools>=57.4.0', 'cython'] build-backend = 'poetry.core.masonry.api' From 57707c3cbcc4ef0e606d8b1d45fe58079eb94bda Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 09:09:01 +0300 Subject: [PATCH 16/35] [EN-3260] install_name_tool --- .github/workflows/buildanduploadartifacts.yml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/buildanduploadartifacts.yml diff --git a/.github/workflows/buildanduploadartifacts.yml b/.github/workflows/buildanduploadartifacts.yml new file mode 100644 index 0000000..e11e1cf --- /dev/null +++ b/.github/workflows/buildanduploadartifacts.yml @@ -0,0 +1,51 @@ +# This workflow publishes package to pypi via poetry + +name: Build and upload artifacts + +on: + push: + tags: + - '*' + workflow_dispatch: + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install poetry + run: | + pip install poetry + shell: bash + - name: Set poetry env + run: | + poetry config virtualenvs.create false + poetry install --no-root + shell: bash + - name: Create tarball + if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7 + run: | + task build -f sdist + shell: bash + - name: Create wheel + run: | + task build -f wheel + shell: bash + - name: Upload artifacts + if: success() + uses: actions/upload-artifact@v2 + with: + name: artifacts + path: dist/ From 826c38b4295333f1f8b560690157a8fd7a48ef78 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 09:16:18 +0300 Subject: [PATCH 17/35] [EN-3260] install_name_tool --- build.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.py b/build.py index 9a15e5b..c0f5818 100644 --- a/build.py +++ b/build.py @@ -96,8 +96,6 @@ elif platform.system() == 'Darwin': runtime_library_dirs = ['.', str(capi_bin_dir)] extra_link_args = None # [f'-Wl,-rpath,@loader_path/dxfeed/core', f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] - # capi_full_library_file_path = capi_bin_dir / capi_library_file_name - # run(('install_name_tool', '-id', f'@rpath/{capi_library_file_name}', str(capi_full_library_file_path))) else: runtime_library_dirs = ['$ORIGIN', '.', str(capi_bin_dir)] extra_link_args = None From d38c9bf9ad7d1b33c44114139f70d7e3049a491c Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 10:41:52 +0300 Subject: [PATCH 18/35] [EN-3260] @executable_path --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index c0f5818..2bf5580 100644 --- a/build.py +++ b/build.py @@ -95,7 +95,7 @@ extra_link_args = None elif platform.system() == 'Darwin': runtime_library_dirs = ['.', str(capi_bin_dir)] - extra_link_args = None # [f'-Wl,-rpath,@loader_path/dxfeed/core', f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] + extra_link_args = ['-Wl,-rpath,@executable_path', '-Wl,-rpath,@loader_path/dxfeed/core', f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] else: runtime_library_dirs = ['$ORIGIN', '.', str(capi_bin_dir)] extra_link_args = None From d2a89d1eef4856b0e02c148343dbf3d952aaf196 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 13:00:22 +0300 Subject: [PATCH 19/35] [EN-3260] install_name_tool --- build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 2bf5580..dbfb7fb 100644 --- a/build.py +++ b/build.py @@ -95,7 +95,8 @@ extra_link_args = None elif platform.system() == 'Darwin': runtime_library_dirs = ['.', str(capi_bin_dir)] - extra_link_args = ['-Wl,-rpath,@executable_path', '-Wl,-rpath,@loader_path/dxfeed/core', f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] + extra_link_args = [f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] + run(('install_name_tool', '-id', f'@rpath/{capi_library_file_name}', capi_library_file_name)) else: runtime_library_dirs = ['$ORIGIN', '.', str(capi_bin_dir)] extra_link_args = None From 0c6adec3a4d8f7e0f2c855ad00aca9b8f692c043 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 13:16:59 +0300 Subject: [PATCH 20/35] [EN-3260] test 8.3.1 --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index dbfb7fb..83eadc3 100644 --- a/build.py +++ b/build.py @@ -25,7 +25,7 @@ root_path = Path(__file__).resolve().parent print(root_path) -capi_version = '8.3.0' +capi_version = '8.3.1' is_x64 = 8 * struct.calcsize("P") == 64 @@ -44,7 +44,7 @@ capi_extracted_root_dir = path_to_extract / f'dxfeed-c-api-{capi_version}-no-tls' if (not os.path.exists(path_to_extract)) or (not os.path.exists(capi_extracted_root_dir)): - url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{capi_version}/dxfeed-c-api-{capi_version}-' \ + url = f'https://github.com/ttldtor/dxfeed-c-api/releases/download/{capi_version}/dxfeed-c-api-{capi_version}-' \ f'{current_os}-no-tls.zip ' print(f'Downloading the "{url}"') resp = urlopen(url) From 6b7da6741481437aaa1beb360023b79559ec7597 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 13:20:56 +0300 Subject: [PATCH 21/35] [EN-3260] test 8.3.1 --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 83eadc3..02b33e4 100644 --- a/build.py +++ b/build.py @@ -96,7 +96,7 @@ elif platform.system() == 'Darwin': runtime_library_dirs = ['.', str(capi_bin_dir)] extra_link_args = [f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] - run(('install_name_tool', '-id', f'@rpath/{capi_library_file_name}', capi_library_file_name)) + # run(('install_name_tool', '-id', f'@rpath/{capi_library_file_name}', capi_library_file_name)) else: runtime_library_dirs = ['$ORIGIN', '.', str(capi_bin_dir)] extra_link_args = None From 248d600f620042b22145e349acc42a245df48d63 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 13:51:44 +0300 Subject: [PATCH 22/35] [EN-3260] test 8.3.1 --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 02b33e4..5b14216 100644 --- a/build.py +++ b/build.py @@ -94,8 +94,8 @@ runtime_library_dirs = None extra_link_args = None elif platform.system() == 'Darwin': - runtime_library_dirs = ['.', str(capi_bin_dir)] - extra_link_args = [f'-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] + runtime_library_dirs = ['@loader_path', '.', str(capi_bin_dir)] + extra_link_args = ['-Wl,-rpath,@loader_path', '-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] # run(('install_name_tool', '-id', f'@rpath/{capi_library_file_name}', capi_library_file_name)) else: runtime_library_dirs = ['$ORIGIN', '.', str(capi_bin_dir)] From ab5cb6c43eea929c6a4ffe6fdb230630e47a1bb1 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 13:58:15 +0300 Subject: [PATCH 23/35] [EN-3260] test 8.3.1 --- build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.py b/build.py index 5b14216..9044d6e 100644 --- a/build.py +++ b/build.py @@ -95,8 +95,7 @@ extra_link_args = None elif platform.system() == 'Darwin': runtime_library_dirs = ['@loader_path', '.', str(capi_bin_dir)] - extra_link_args = ['-Wl,-rpath,@loader_path', '-Wl,-rpath,.', f'-Wl,-rpath,{str(capi_bin_dir)}'] - # run(('install_name_tool', '-id', f'@rpath/{capi_library_file_name}', capi_library_file_name)) + extra_link_args = ['-Wl,-rpath,@loader_path'] else: runtime_library_dirs = ['$ORIGIN', '.', str(capi_bin_dir)] extra_link_args = None From 0720ff70ca8d5f8c49d0075d03361f6114e15eee Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 14:03:13 +0300 Subject: [PATCH 24/35] [EN-3260] -runtime_library_dirs --- .github/workflows/buildanduploadartifacts.yml | 51 ------------------- build.py | 2 +- 2 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 .github/workflows/buildanduploadartifacts.yml diff --git a/.github/workflows/buildanduploadartifacts.yml b/.github/workflows/buildanduploadartifacts.yml deleted file mode 100644 index e11e1cf..0000000 --- a/.github/workflows/buildanduploadartifacts.yml +++ /dev/null @@ -1,51 +0,0 @@ -# This workflow publishes package to pypi via poetry - -name: Build and upload artifacts - -on: - push: - tags: - - '*' - workflow_dispatch: - -jobs: - build: - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.6, 3.7, 3.8, 3.9] - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install poetry - run: | - pip install poetry - shell: bash - - name: Set poetry env - run: | - poetry config virtualenvs.create false - poetry install --no-root - shell: bash - - name: Create tarball - if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7 - run: | - task build -f sdist - shell: bash - - name: Create wheel - run: | - task build -f wheel - shell: bash - - name: Upload artifacts - if: success() - uses: actions/upload-artifact@v2 - with: - name: artifacts - path: dist/ diff --git a/build.py b/build.py index 9044d6e..e71db95 100644 --- a/build.py +++ b/build.py @@ -94,7 +94,7 @@ runtime_library_dirs = None extra_link_args = None elif platform.system() == 'Darwin': - runtime_library_dirs = ['@loader_path', '.', str(capi_bin_dir)] + runtime_library_dirs = None extra_link_args = ['-Wl,-rpath,@loader_path'] else: runtime_library_dirs = ['$ORIGIN', '.', str(capi_bin_dir)] From 70f2936b7ac89dd3bc7da3e343db35b797457fea Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Sun, 29 Aug 2021 14:45:32 +0300 Subject: [PATCH 25/35] [EN-3260] fix get_include(), fix c api version retrieving, fix the file names --- build.py | 117 +++++++++++++++------------------- dxfeed/core/utils/helpers.pyx | 10 +-- 2 files changed, 58 insertions(+), 69 deletions(-) diff --git a/build.py b/build.py index e71db95..9fac6bd 100644 --- a/build.py +++ b/build.py @@ -1,18 +1,18 @@ import os -import sys import shutil import struct -from subprocess import run from setuptools import Extension, find_packages from setuptools.dist import Distribution -from setuptools.command.build_ext import build_ext from pathlib import Path from io import BytesIO from zipfile import ZipFile from urllib.request import urlopen import platform +DEFAULT_C_API_VERSION = '8.3.0' +c_api_version = os.getenv('DXFEED_C_API_VERSION', DEFAULT_C_API_VERSION) + try: from Cython.Build import cythonize except ImportError: @@ -23,9 +23,6 @@ ext = 'pyx' root_path = Path(__file__).resolve().parent -print(root_path) - -capi_version = '8.3.1' is_x64 = 8 * struct.calcsize("P") == 64 @@ -34,17 +31,16 @@ elif platform.system() == 'Darwin': current_os = 'macosx' else: - current_os = 'centos' + current_os = 'centos' # Since centos uses an earlier version of libc -capi_root_dir = root_path / 'dxfeed' / 'dxfeed-c-api' -capi_include_dir = capi_root_dir / 'include' -# capi_bin_dir = capi_root_dir / 'bin' -capi_bin_dir = root_path / 'dxfeed' / 'core' +c_api_root_dir = root_path / 'dxfeed' / 'dxfeed-c-api' +c_api_include_dir = c_api_root_dir / 'include' +c_api_bin_dir = root_path / 'dxfeed' / 'core' path_to_extract = root_path / 'dxfeed' / 'tmp' -capi_extracted_root_dir = path_to_extract / f'dxfeed-c-api-{capi_version}-no-tls' +c_api_extracted_root_dir = path_to_extract / f'dxfeed-c-api-{c_api_version}-no-tls' -if (not os.path.exists(path_to_extract)) or (not os.path.exists(capi_extracted_root_dir)): - url = f'https://github.com/ttldtor/dxfeed-c-api/releases/download/{capi_version}/dxfeed-c-api-{capi_version}-' \ +if (not os.path.exists(path_to_extract)) or (not os.path.exists(c_api_extracted_root_dir)): + url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{c_api_version}/dxfeed-c-api-{c_api_version}-' \ f'{current_os}-no-tls.zip ' print(f'Downloading the "{url}"') resp = urlopen(url) @@ -52,43 +48,53 @@ print(f'Extracting to "{path_to_extract}"') zipfile.extractall(path_to_extract) +if __debug__: + # noinspection PyUnreachableCode + c_api_debug_suffix = 'd' +else: + c_api_debug_suffix = '' + +if is_x64: + c_api_x64_suffix = '_64' +else: + c_api_x64_suffix = '' + +c_api_library_name = f'DXFeed{c_api_debug_suffix}{c_api_x64_suffix}' + if platform.system() == 'Windows': if is_x64: - capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x64' - capi_library_name = 'DXFeed_64' - capi_library_file_name = f'{capi_library_name}.dll' - capi_library_file_name2 = f'{capi_library_name}.lib' + c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x64' + c_api_library_file_name = f'{c_api_library_name}.dll' + c_api_library_file_name2 = f'{c_api_library_name}.lib' else: - capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x86' - capi_library_name = 'DXFeed' - capi_library_file_name = f'{capi_library_name}.dll' - capi_library_file_name2 = f'{capi_library_name}.lib' + c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x86' + c_api_library_file_name = f'{c_api_library_name}.dll' + c_api_library_file_name2 = f'{c_api_library_name}.lib' elif platform.system() == 'Darwin': if is_x64: - capi_extracted_root_dir = capi_extracted_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls' - capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x64' - capi_library_name = 'DXFeed_64' - capi_library_file_name = f'lib{capi_library_name}.dylib' + c_api_extracted_root_dir = c_api_extracted_root_dir / f'DXFeedAll-{c_api_version}-x64-no-tls' + c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x64' + c_api_library_file_name = f'lib{c_api_library_name}.dylib' else: raise Exception('Unsupported platform') else: if is_x64: - capi_extracted_root_dir = capi_extracted_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls' - capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x64' - capi_library_name = 'DXFeed_64' - capi_library_file_name = f'lib{capi_library_name}.so' + c_api_extracted_root_dir = c_api_extracted_root_dir / f'DXFeedAll-{c_api_version}-x64-no-tls' + c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x64' + c_api_library_name = 'DXFeed_64' + c_api_library_file_name = f'lib{c_api_library_name}.so' else: raise Exception('Unsupported platform') -if not os.path.exists(capi_include_dir): - shutil.copytree(capi_extracted_root_dir / 'include', capi_include_dir) -if not os.path.exists(capi_bin_dir / capi_library_file_name): - # os.makedirs(capi_bin_dir) - shutil.copyfile(capi_extracted_library_dir / capi_library_file_name, capi_bin_dir / capi_library_file_name) +if not os.path.exists(c_api_include_dir): + shutil.copytree(c_api_extracted_root_dir / 'include', c_api_include_dir) +if not os.path.exists(c_api_bin_dir / c_api_library_file_name): + # os.makedirs(c_api_bin_dir) + shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name, c_api_bin_dir / c_api_library_file_name) if platform.system() == 'Windows': # noinspection PyUnboundLocalVariable - if not os.path.exists(capi_bin_dir / capi_library_file_name2): - shutil.copyfile(capi_extracted_library_dir / capi_library_file_name2, capi_bin_dir / capi_library_file_name2) + if not os.path.exists(c_api_bin_dir / c_api_library_file_name2): + shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name2, c_api_bin_dir / c_api_library_file_name2) if platform.system() == 'Windows': runtime_library_dirs = None @@ -97,56 +103,37 @@ runtime_library_dirs = None extra_link_args = ['-Wl,-rpath,@loader_path'] else: - runtime_library_dirs = ['$ORIGIN', '.', str(capi_bin_dir)] + runtime_library_dirs = ['$ORIGIN'] extra_link_args = None -capi_include_dirs = [str(capi_include_dir)] +c_api_include_dirs = [str(c_api_include_dir)] -libs = [capi_library_name] +libs = [c_api_library_name] if platform.system() == 'Windows': libs.append('ws2_32') extensions = [Extension('dxfeed.core.utils.helpers', ['dxfeed/core/utils/helpers.' + ext], - include_dirs=capi_include_dirs), + include_dirs=c_api_include_dirs), Extension('dxfeed.core.utils.handler', ['dxfeed/core/utils/handler.' + ext], - include_dirs=capi_include_dirs), + include_dirs=c_api_include_dirs), Extension('dxfeed.core.listeners.listener', ['dxfeed/core/listeners/listener.' + ext], - include_dirs=capi_include_dirs), - Extension('dxfeed.core.DXFeedPy', ['dxfeed/core/DXFeedPy.' + ext], library_dirs=[str(capi_bin_dir)], + include_dirs=c_api_include_dirs), + Extension('dxfeed.core.DXFeedPy', ['dxfeed/core/DXFeedPy.' + ext], library_dirs=[str(c_api_bin_dir)], runtime_library_dirs=runtime_library_dirs, extra_link_args=extra_link_args, libraries=libs, - include_dirs=capi_include_dirs)] + include_dirs=c_api_include_dirs)] if use_cython: extensions = cythonize(extensions, language_level=3) -class custom_build_ext(build_ext): - """ - Add appropriate rpath linker flags (necessary on mac) - """ - - def finalize_options(self): - super().finalize_options() - # Special treatment of rpath in case of OSX, to work around python - # distutils bug 36353. This constructs proper rpath arguments for clang. - # See https://bugs.python.org/issue36353 - # Workaround from https://github.com/python/cpython/pull/12418 - if sys.platform[:6] == "darwin": - for path in self.rpath: - for ext in self.extensions: - ext.extra_link_args.append("-Wl,-rpath," + path) - self.rpath[:] = [] - - def build(setup_kwargs): setup_kwargs.update({ - 'cmdclass': {'build_ext': custom_build_ext}, 'ext_modules': extensions, 'zip_safe': False, 'packages': find_packages(), - 'include_dirs': capi_include_dirs + 'include_dirs': c_api_include_dirs }) diff --git a/dxfeed/core/utils/helpers.pyx b/dxfeed/core/utils/helpers.pyx index c1fd9ec..2d99f92 100644 --- a/dxfeed/core/utils/helpers.pyx +++ b/dxfeed/core/utils/helpers.pyx @@ -1,3 +1,4 @@ +import os from warnings import warn from dxfeed.core.pxd_include.DXErrorCodes cimport * from dxfeed.core.pxd_include.DXFeed cimport * @@ -22,8 +23,8 @@ cdef object unicode_from_dxf_const_string_t(dxf_const_string_t wcs, int size=-1) Python unicode string """ if wcs == NULL: - return '' - ret_val = PyUnicode_FromWideChar(wcs, size) + return '' + ret_val = PyUnicode_FromWideChar(wcs, size) return ret_val cdef dxf_const_string_t dxf_const_string_t_from_unicode(object symbol): @@ -88,5 +89,6 @@ def get_include(): List of paths to header files """ out_dir = list() - out_dir.append(str(Path(dxfeed.__file__).resolve().parent.joinpath('dxfeed-c-api', 'include'))) - return out_dir \ No newline at end of file + out_dir.append( + str(os.path.realpath(Path(dxfeed.__file__).resolve().parent / '..' / 'libs' / 'dxfeed-c-api' / 'include'))) + return out_dir From 1c8eb4d27f7c9a8a5df0b56338af0e875577c2f8 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Mon, 30 Aug 2021 15:40:04 +0300 Subject: [PATCH 26/35] [EN-3260] add Downloader, build.native-dependencies, ubuntu-18.04, test on PR --- .github/workflows/publishpackage.yml | 7 +- .github/workflows/test_on_pr.yml | 76 ++++++++++++ .github/workflows/testdevelop.yml | 2 +- .github/workflows/testpackage.yml | 4 +- build.py | 174 +++++++++++++++------------ clear.py | 3 - dxfeed/core/utils/helpers.pyx | 2 +- pyproject.toml | 10 +- 8 files changed, 187 insertions(+), 91 deletions(-) create mode 100644 .github/workflows/test_on_pr.yml diff --git a/.github/workflows/publishpackage.yml b/.github/workflows/publishpackage.yml index d985ac4..58982a5 100644 --- a/.github/workflows/publishpackage.yml +++ b/.github/workflows/publishpackage.yml @@ -11,8 +11,8 @@ jobs: build: strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.6, 3.7, 3.8] + os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest] + python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} @@ -34,12 +34,11 @@ jobs: poetry install --no-root shell: bash - name: Create tarball - if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7 + if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-18.04' run: | task build -f sdist shell: bash - name: Create wheel - if: matrix.os == 'windows-latest' || matrix.os == 'macos-latest' run: | task build -f wheel shell: bash diff --git a/.github/workflows/test_on_pr.yml b/.github/workflows/test_on_pr.yml new file mode 100644 index 0000000..ea30ba5 --- /dev/null +++ b/.github/workflows/test_on_pr.yml @@ -0,0 +1,76 @@ +# This workflow will check package installation and runs tests on PR +name: Test on PR + +on: + pull_request: + types: [opened, reopened, synchronize] + +jobs: + installation: + strategy: + matrix: + os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install poetry + run: | + pip install poetry + shell: bash + - name: Set poetry env + run: | + poetry config virtualenvs.create false + poetry install --no-root --no-dev + pip install taskipy cython + task build -f sdist + pip uninstall --yes taskipy cython + shell: bash + - name: Install package artifact + run: | + pip install dist/dxfeed* + pip uninstall --yes dxfeed + shell: bash + tests: + needs: installation + strategy: + matrix: + os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install poetry + run: | + pip install poetry + shell: bash + - name: Set poetry env + run: | + poetry config virtualenvs.create false + poetry install --no-root + shell: bash + - name: Run tests + run: | + task test + shell: bash + - name: Generate doc + run: | + task html_docs + shell: bash + if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7 diff --git a/.github/workflows/testdevelop.yml b/.github/workflows/testdevelop.yml index 8706b04..6c13fa8 100644 --- a/.github/workflows/testdevelop.yml +++ b/.github/workflows/testdevelop.yml @@ -12,7 +12,7 @@ jobs: installation: strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/testpackage.yml b/.github/workflows/testpackage.yml index 6d90cfc..2c63154 100644 --- a/.github/workflows/testpackage.yml +++ b/.github/workflows/testpackage.yml @@ -13,8 +13,8 @@ jobs: installation: strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.6, 3.7, 3.8] + os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest] + python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} diff --git a/build.py b/build.py index 9fac6bd..a662771 100644 --- a/build.py +++ b/build.py @@ -1,17 +1,96 @@ import os -import shutil +import platform import struct - -from setuptools import Extension, find_packages -from setuptools.dist import Distribution -from pathlib import Path from io import BytesIO -from zipfile import ZipFile +from pathlib import Path from urllib.request import urlopen -import platform +from zipfile import ZipFile +import shutil +from setuptools import Extension, find_packages +from setuptools.dist import Distribution +import toml + + +class Downloader(object): + def __init__(self, version: str, path_to_extract: Path, + path_to_copy_includes: Path, path_to_copy_libs: Path): + self.__version = version + self.__path_to_extract = path_to_extract + self.__path_to_copy_includes = path_to_copy_includes + self.__path_to_copy_libs = path_to_copy_libs + + def download(self) -> str: + __c_api_extracted_root_dir = self.__path_to_extract / f'dxfeed-c-api-{self.__version}-no-tls' + is_x64 = 8 * struct.calcsize("P") == 64 + + if platform.system() == 'Windows': + current_os = 'windows' + elif platform.system() == 'Darwin': + current_os = 'macosx' + else: + current_os = 'centos' # Since centos uses an earlier version of libc + + if (not os.path.exists(self.__path_to_extract)) or (not os.path.exists(__c_api_extracted_root_dir)): + url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{self.__version}/dxfeed-c-api-' \ + f'{self.__version}-{current_os}-no-tls.zip ' + print(f'Downloading the "{url}"') + resp = urlopen(url) + zipfile = ZipFile(BytesIO(resp.read())) + print(f'Extracting to "{self.__path_to_extract}"') + zipfile.extractall(self.__path_to_extract) + + if __debug__: + # noinspection PyUnreachableCode + c_api_debug_suffix = 'd' + else: + c_api_debug_suffix = '' + + if is_x64: + c_api_x64_suffix = '_64' + else: + c_api_x64_suffix = '' + + __c_api_library_name = f'DXFeed{c_api_debug_suffix}{c_api_x64_suffix}' + + if platform.system() == 'Windows': + if is_x64: + c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x64' + c_api_library_file_name = f'{__c_api_library_name}.dll' + c_api_library_file_name2 = f'{__c_api_library_name}.lib' + else: + c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x86' + c_api_library_file_name = f'{__c_api_library_name}.dll' + c_api_library_file_name2 = f'{__c_api_library_name}.lib' + elif platform.system() == 'Darwin': + if is_x64: + __c_api_extracted_root_dir = __c_api_extracted_root_dir / f'DXFeedAll-{self.__version}-x64-no-tls' + c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x64' + c_api_library_file_name = f'lib{__c_api_library_name}.dylib' + else: + raise Exception('Unsupported platform') + else: + if is_x64: + __c_api_extracted_root_dir = __c_api_extracted_root_dir / f'DXFeedAll-{self.__version}-x64-no-tls' + c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x64' + c_api_library_file_name = f'lib{__c_api_library_name}.so' + else: + raise Exception('Unsupported platform') + + if not os.path.exists(self.__path_to_copy_includes): + shutil.copytree(__c_api_extracted_root_dir / 'include', self.__path_to_copy_includes) + if not os.path.exists(self.__path_to_copy_libs / c_api_library_file_name): + if not os.path.exists(self.__path_to_copy_libs): + os.makedirs(self.__path_to_copy_libs) + shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name, + self.__path_to_copy_libs / c_api_library_file_name) + if platform.system() == 'Windows': + # noinspection PyUnboundLocalVariable + if not os.path.exists(self.__path_to_copy_libs / c_api_library_file_name2): + shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name2, + self.__path_to_copy_libs / c_api_library_file_name2) + + return __c_api_library_name -DEFAULT_C_API_VERSION = '8.3.0' -c_api_version = os.getenv('DXFEED_C_API_VERSION', DEFAULT_C_API_VERSION) try: from Cython.Build import cythonize @@ -24,77 +103,18 @@ root_path = Path(__file__).resolve().parent -is_x64 = 8 * struct.calcsize("P") == 64 - -if platform.system() == 'Windows': - current_os = 'windows' -elif platform.system() == 'Darwin': - current_os = 'macosx' -else: - current_os = 'centos' # Since centos uses an earlier version of libc +pyproject = toml.load(root_path / 'pyproject.toml') +c_api_version = pyproject['build']['native-dependencies']['dxfeed_c_api'] +if c_api_version == 'env': + c_api_version = os.getenv('DXFEED_C_API_VERSION') c_api_root_dir = root_path / 'dxfeed' / 'dxfeed-c-api' +path_to_extract = root_path / 'dxfeed' / 'tmp' c_api_include_dir = c_api_root_dir / 'include' c_api_bin_dir = root_path / 'dxfeed' / 'core' -path_to_extract = root_path / 'dxfeed' / 'tmp' -c_api_extracted_root_dir = path_to_extract / f'dxfeed-c-api-{c_api_version}-no-tls' - -if (not os.path.exists(path_to_extract)) or (not os.path.exists(c_api_extracted_root_dir)): - url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{c_api_version}/dxfeed-c-api-{c_api_version}-' \ - f'{current_os}-no-tls.zip ' - print(f'Downloading the "{url}"') - resp = urlopen(url) - zipfile = ZipFile(BytesIO(resp.read())) - print(f'Extracting to "{path_to_extract}"') - zipfile.extractall(path_to_extract) - -if __debug__: - # noinspection PyUnreachableCode - c_api_debug_suffix = 'd' -else: - c_api_debug_suffix = '' - -if is_x64: - c_api_x64_suffix = '_64' -else: - c_api_x64_suffix = '' -c_api_library_name = f'DXFeed{c_api_debug_suffix}{c_api_x64_suffix}' - -if platform.system() == 'Windows': - if is_x64: - c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x64' - c_api_library_file_name = f'{c_api_library_name}.dll' - c_api_library_file_name2 = f'{c_api_library_name}.lib' - else: - c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x86' - c_api_library_file_name = f'{c_api_library_name}.dll' - c_api_library_file_name2 = f'{c_api_library_name}.lib' -elif platform.system() == 'Darwin': - if is_x64: - c_api_extracted_root_dir = c_api_extracted_root_dir / f'DXFeedAll-{c_api_version}-x64-no-tls' - c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x64' - c_api_library_file_name = f'lib{c_api_library_name}.dylib' - else: - raise Exception('Unsupported platform') -else: - if is_x64: - c_api_extracted_root_dir = c_api_extracted_root_dir / f'DXFeedAll-{c_api_version}-x64-no-tls' - c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x64' - c_api_library_name = 'DXFeed_64' - c_api_library_file_name = f'lib{c_api_library_name}.so' - else: - raise Exception('Unsupported platform') - -if not os.path.exists(c_api_include_dir): - shutil.copytree(c_api_extracted_root_dir / 'include', c_api_include_dir) -if not os.path.exists(c_api_bin_dir / c_api_library_file_name): - # os.makedirs(c_api_bin_dir) - shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name, c_api_bin_dir / c_api_library_file_name) -if platform.system() == 'Windows': - # noinspection PyUnboundLocalVariable - if not os.path.exists(c_api_bin_dir / c_api_library_file_name2): - shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name2, c_api_bin_dir / c_api_library_file_name2) +downloader = Downloader(c_api_version, path_to_extract, c_api_include_dir, c_api_bin_dir) +c_api_library_name = downloader.download() if platform.system() == 'Windows': runtime_library_dirs = None @@ -152,7 +172,3 @@ def build_extensions(): build_ext_cmd.ensure_finalized() build_ext_cmd.inplace = 1 build_ext_cmd.run() - - print(f'Removing the "{path_to_extract}"') - if os.path.exists(path_to_extract): - shutil.rmtree(path_to_extract) diff --git a/clear.py b/clear.py index e467611..1ae53a8 100644 --- a/clear.py +++ b/clear.py @@ -16,8 +16,5 @@ if os.path.exists('dxfeed/tmp'): shutil.rmtree('dxfeed/tmp') -if os.path.exists('dxfeed/dxfeed-c-api/bin'): - shutil.rmtree('dxfeed/dxfeed-c-api/bin') - if os.path.exists('dxfeed/dxfeed-c-api/include'): shutil.rmtree('dxfeed/dxfeed-c-api/include') diff --git a/dxfeed/core/utils/helpers.pyx b/dxfeed/core/utils/helpers.pyx index 2d99f92..338fae6 100644 --- a/dxfeed/core/utils/helpers.pyx +++ b/dxfeed/core/utils/helpers.pyx @@ -90,5 +90,5 @@ def get_include(): """ out_dir = list() out_dir.append( - str(os.path.realpath(Path(dxfeed.__file__).resolve().parent / '..' / 'libs' / 'dxfeed-c-api' / 'include'))) + str(os.path.realpath(Path(dxfeed.__file__).resolve().parent / 'dxfeed-c-api' / 'include'))) return out_dir diff --git a/pyproject.toml b/pyproject.toml index b4f6f1f..507392c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,10 @@ classifiers = ['Development Status :: 4 - Beta', 'Operating System :: MacOS', 'Programming Language :: Cython', 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Office/Business :: Financial', 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', 'Natural Language :: English'] @@ -42,6 +46,7 @@ pytest = '^6.2.4' sphinx = '^4.1.2' sphinx_rtd_theme = '^0.5.2' pygments = '^2.10' +toml = '^0.10.2' [tool.poetry.extras] docs = ['jupyter', 'cython'] @@ -56,5 +61,8 @@ post_build = 'task clear' post_test = 'task clear' [build-system] -requires = ['poetry_core>=1.0.0', 'setuptools>=57.4.0', 'cython'] +requires = ['poetry_core>=1.0.0', 'setuptools>=57.4.0', 'cython', 'toml>=0.10.2'] build-backend = 'poetry.core.masonry.api' + +[build.native-dependencies] +dxfeed_c_api = '8.3.0' From f80070b0aa84495312b143d424e1462c691aa3c2 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Tue, 31 Aug 2021 22:16:57 +0300 Subject: [PATCH 27/35] [EN-3260] -debug version --- build.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/build.py b/build.py index a662771..85f53b7 100644 --- a/build.py +++ b/build.py @@ -39,18 +39,12 @@ def download(self) -> str: print(f'Extracting to "{self.__path_to_extract}"') zipfile.extractall(self.__path_to_extract) - if __debug__: - # noinspection PyUnreachableCode - c_api_debug_suffix = 'd' - else: - c_api_debug_suffix = '' - if is_x64: c_api_x64_suffix = '_64' else: c_api_x64_suffix = '' - __c_api_library_name = f'DXFeed{c_api_debug_suffix}{c_api_x64_suffix}' + __c_api_library_name = f'DXFeed{c_api_x64_suffix}' if platform.system() == 'Windows': if is_x64: From bbb136bc61fe5b728347fd99f89c564e94eb0cfe Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Fri, 3 Sep 2021 13:11:40 +0300 Subject: [PATCH 28/35] [EN-3260] fix clear --- clear.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/clear.py b/clear.py index 1ae53a8..a295b4e 100644 --- a/clear.py +++ b/clear.py @@ -1,20 +1,20 @@ -import glob -import os import shutil +from pathlib import Path +from itertools import chain -c_files = glob.glob('dxfeed/core/**/*.c', recursive=True) -cpp_files = glob.glob('dxfeed/core/**/*.cpp', recursive=True) -pyd_files = glob.glob('dxfeed/core/**/*.pyd', recursive=True) -so_files = glob.glob('dxfeed/core/**/*.so', recursive=True) -dll_files = glob.glob('dxfeed/core/**/*.dll', recursive=True) -lib_files = glob.glob('dxfeed/core/**/*.lib', recursive=True) -dylib_files = glob.glob('dxfeed/core/**/*.dylib', recursive=True) +c_files = Path('dxfeed/core').glob('**/*.c') +cpp_files = Path('dxfeed/core').glob('**/*.cpp') +pyd_files = Path('dxfeed/core').glob('**/*.pyd') +so_files = Path('dxfeed/core').glob('**/*.so') +dll_files = Path('dxfeed/core').glob('**/*.dll') +lib_files = Path('dxfeed/core').glob('*.lib') +dylib_files = Path('dxfeed/core').glob('**/*.dylib') -for file_path in c_files + cpp_files + pyd_files + so_files + dll_files + lib_files + dylib_files: - os.remove(file_path) +for file_path in chain(c_files, cpp_files, pyd_files, so_files, dll_files, lib_files, dylib_files): + file_path.unlink() -if os.path.exists('dxfeed/tmp'): +if Path('dxfeed/tmp').exists(): shutil.rmtree('dxfeed/tmp') -if os.path.exists('dxfeed/dxfeed-c-api/include'): +if Path('dxfeed/dxfeed-c-api/include').exists(): shutil.rmtree('dxfeed/dxfeed-c-api/include') From 04211d91c792cd65017ae02d886179112067bf21 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Fri, 3 Sep 2021 13:12:02 +0300 Subject: [PATCH 29/35] [EN-3260] fix workflows --- .github/workflows/publishpackage.yml | 6 +++--- .github/workflows/test_on_pr.yml | 6 +++--- .github/workflows/testdevelop.yml | 6 +++--- .github/workflows/testpackage.yml | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/publishpackage.yml b/.github/workflows/publishpackage.yml index 58982a5..199a900 100644 --- a/.github/workflows/publishpackage.yml +++ b/.github/workflows/publishpackage.yml @@ -11,7 +11,7 @@ jobs: build: strategy: matrix: - os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-latest, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} @@ -34,7 +34,7 @@ jobs: poetry install --no-root shell: bash - name: Create tarball - if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-18.04' + if: matrix.python-version == 3.7 run: | task build -f sdist shell: bash @@ -51,7 +51,7 @@ jobs: publish: - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 needs: build steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/test_on_pr.yml b/.github/workflows/test_on_pr.yml index ea30ba5..ad393e3 100644 --- a/.github/workflows/test_on_pr.yml +++ b/.github/workflows/test_on_pr.yml @@ -9,7 +9,7 @@ jobs: installation: strategy: matrix: - os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-latest, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} @@ -43,7 +43,7 @@ jobs: needs: installation strategy: matrix: - os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-latest, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} @@ -73,4 +73,4 @@ jobs: run: | task html_docs shell: bash - if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7 + if: matrix.os == 'ubuntu-18.04' && matrix.python-version == 3.7 diff --git a/.github/workflows/testdevelop.yml b/.github/workflows/testdevelop.yml index 6c13fa8..1d4f1b2 100644 --- a/.github/workflows/testdevelop.yml +++ b/.github/workflows/testdevelop.yml @@ -12,7 +12,7 @@ jobs: installation: strategy: matrix: - os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-latest, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} @@ -46,7 +46,7 @@ jobs: needs: installation strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-latest, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} @@ -76,4 +76,4 @@ jobs: run: | task html_docs shell: bash - if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7 + if: matrix.os == 'ubuntu-18.04' && matrix.python-version == 3.7 diff --git a/.github/workflows/testpackage.yml b/.github/workflows/testpackage.yml index 2c63154..1603b3a 100644 --- a/.github/workflows/testpackage.yml +++ b/.github/workflows/testpackage.yml @@ -13,7 +13,7 @@ jobs: installation: strategy: matrix: - os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-latest, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} @@ -47,7 +47,7 @@ jobs: needs: installation strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-latest, macos-latest] python-version: [3.6, 3.7, 3.8] runs-on: ${{ matrix.os }} @@ -77,4 +77,4 @@ jobs: run: | task html_docs shell: bash - if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7 + if: matrix.os == 'ubuntu-18.04' && matrix.python-version == 3.7 From 27e18a132e1954c6f879b02ab893d18c31793c9c Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Fri, 3 Sep 2021 14:45:11 +0300 Subject: [PATCH 30/35] [EN-3260] docs, comments, fixes --- build.py | 140 +++++++++++++++++++++++++++++++------------------ pyproject.toml | 4 ++ 2 files changed, 92 insertions(+), 52 deletions(-) diff --git a/build.py b/build.py index 85f53b7..d520794 100644 --- a/build.py +++ b/build.py @@ -12,78 +12,114 @@ class Downloader(object): + """Class that downloads a dxFeed C API bundle and places the necessary include or lib files""" + def __init__(self, version: str, path_to_extract: Path, path_to_copy_includes: Path, path_to_copy_libs: Path): + """ + Parameters + ---------- + version : str + The dxFeed C API version + path_to_extract : Path + The temporary directory where the bundle should be unpacked + path_to_copy_includes : Path + The directory where the include files should be placed + path_to_copy_libs + The directory where the library files should be placed + """ + self.__version = version self.__path_to_extract = path_to_extract self.__path_to_copy_includes = path_to_copy_includes self.__path_to_copy_libs = path_to_copy_libs - def download(self) -> str: - __c_api_extracted_root_dir = self.__path_to_extract / f'dxfeed-c-api-{self.__version}-no-tls' - is_x64 = 8 * struct.calcsize("P") == 64 + def download(self, dxfeed_c_api_bundle_url_template: str) -> str: + """ + The method that does the bulk of the work of downloading and placement files. - if platform.system() == 'Windows': - current_os = 'windows' - elif platform.system() == 'Darwin': - current_os = 'macosx' - else: - current_os = 'centos' # Since centos uses an earlier version of libc - if (not os.path.exists(self.__path_to_extract)) or (not os.path.exists(__c_api_extracted_root_dir)): - url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{self.__version}/dxfeed-c-api-' \ - f'{self.__version}-{current_os}-no-tls.zip ' + Parameters + ---------- + dxfeed_c_api_bundle_url_template: str + The URL template for zipped dxFeed C API bundle. Parameters used: + {version} - The C API version + {os} - The target OS + + Returns + ------- + : str + The resulting name of the library, which is required to build an extension + + """ + c_api_extracted_root_path = self.__path_to_extract / f'dxfeed-c-api-{self.__version}-no-tls' + is_x64 = 8 * struct.calcsize('P') == 64 + + url_template_os = 'centos' # Since centos uses an earlier version of libc + + if platform.system() == 'Darwin': + url_template_os = 'macosx' + elif platform.system() == 'Windows': + url_template_os = 'windows' + + if (not os.path.exists(self.__path_to_extract)) or (not os.path.exists(c_api_extracted_root_path)): + url = dxfeed_c_api_bundle_url_template.format(version=self.__version, os=url_template_os) print(f'Downloading the "{url}"') resp = urlopen(url) zipfile = ZipFile(BytesIO(resp.read())) print(f'Extracting to "{self.__path_to_extract}"') zipfile.extractall(self.__path_to_extract) - if is_x64: - c_api_x64_suffix = '_64' - else: - c_api_x64_suffix = '' + c_api_x64_suffix = '_64' if is_x64 else '' + resulting_c_api_library_name = f'DXFeed{c_api_x64_suffix}' - __c_api_library_name = f'DXFeed{c_api_x64_suffix}' + # Determine and fixing paths for unpacked artifacts. + # The workaround is related to the packaging feature on POSIX systems in the dxFeed API' CI\CD. + if is_x64: + if platform.system() != 'Windows': + c_api_extracted_root_path = c_api_extracted_root_path / f'DXFeedAll-{self.__version}-x64-no-tls' - if platform.system() == 'Windows': - if is_x64: - c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x64' - c_api_library_file_name = f'{__c_api_library_name}.dll' - c_api_library_file_name2 = f'{__c_api_library_name}.lib' - else: - c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x86' - c_api_library_file_name = f'{__c_api_library_name}.dll' - c_api_library_file_name2 = f'{__c_api_library_name}.lib' - elif platform.system() == 'Darwin': - if is_x64: - __c_api_extracted_root_dir = __c_api_extracted_root_dir / f'DXFeedAll-{self.__version}-x64-no-tls' - c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x64' - c_api_library_file_name = f'lib{__c_api_library_name}.dylib' - else: - raise Exception('Unsupported platform') + c_api_extracted_library_path = c_api_extracted_root_path / 'bin' / 'x64' else: - if is_x64: - __c_api_extracted_root_dir = __c_api_extracted_root_dir / f'DXFeedAll-{self.__version}-x64-no-tls' - c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x64' - c_api_library_file_name = f'lib{__c_api_library_name}.so' + if platform.system() == 'Windows': + c_api_extracted_library_path = c_api_extracted_root_path / 'bin' / 'x86' else: - raise Exception('Unsupported platform') + raise RuntimeError('Unsupported platform') + + # Determine possible prefixes and extensions for library files + lib_file_name_prefixes = [''] * 2 + lib_file_name_extensions = ['dll', 'lib'] + + if platform.system() != 'Windows': + lib_file_name_prefixes = ['lib'] + lib_file_name_extensions = ['dylib'] if platform.system() == 'Darwin' else ['so'] + + # Create paths for libraries to be copied + c_api_library_file_source_paths = [] + c_api_library_file_dest_paths = [] + + for idx, prefix in enumerate(lib_file_name_prefixes): + file_name = f'{prefix}{resulting_c_api_library_name}.{lib_file_name_extensions[idx]}' + c_api_library_file_source_paths.append(c_api_extracted_library_path / file_name) + c_api_library_file_dest_paths.append(self.__path_to_copy_libs / file_name) + + print('Copying all headers') + # Copying all headers + if not Path(self.__path_to_copy_includes).exists(): + shutil.copytree(c_api_extracted_root_path / 'include', self.__path_to_copy_includes) + + # Create a directory where we will copy libraries, if necessary + if not Path(self.__path_to_copy_libs).exists(): + print(f'Creating the {self.__path_to_copy_libs} path') + os.makedirs(self.__path_to_copy_libs) - if not os.path.exists(self.__path_to_copy_includes): - shutil.copytree(__c_api_extracted_root_dir / 'include', self.__path_to_copy_includes) - if not os.path.exists(self.__path_to_copy_libs / c_api_library_file_name): - if not os.path.exists(self.__path_to_copy_libs): - os.makedirs(self.__path_to_copy_libs) - shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name, - self.__path_to_copy_libs / c_api_library_file_name) - if platform.system() == 'Windows': - # noinspection PyUnboundLocalVariable - if not os.path.exists(self.__path_to_copy_libs / c_api_library_file_name2): - shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name2, - self.__path_to_copy_libs / c_api_library_file_name2) + print('Copying the required libraries') + # Copying the required libraries + for idx, path in enumerate(c_api_library_file_source_paths): + print(f' {path} -> {c_api_library_file_dest_paths[idx]}') + shutil.copyfile(path, c_api_library_file_dest_paths[idx]) - return __c_api_library_name + return resulting_c_api_library_name try: @@ -108,7 +144,7 @@ def download(self) -> str: c_api_bin_dir = root_path / 'dxfeed' / 'core' downloader = Downloader(c_api_version, path_to_extract, c_api_include_dir, c_api_bin_dir) -c_api_library_name = downloader.download() +c_api_library_name = downloader.download(pyproject['build']['native-dependencies']['dxfeed_c_api_bundle_url_template']) if platform.system() == 'Windows': runtime_library_dirs = None diff --git a/pyproject.toml b/pyproject.toml index 507392c..d2e7225 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,3 +66,7 @@ build-backend = 'poetry.core.masonry.api' [build.native-dependencies] dxfeed_c_api = '8.3.0' +# The URL template for zipped dxFeed C API bundle. Parameters used: +# {version} - The C API version +# {os} - The target OS +dxfeed_c_api_bundle_url_template = 'https://github.com/dxFeed/dxfeed-c-api/releases/download/{version}/dxfeed-c-api-{version}-{os}-no-tls.zip' From 3b704e57cae706ac30b801057d7000b989387736 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Fri, 3 Sep 2021 14:47:10 +0300 Subject: [PATCH 31/35] [EN-3260] version 0.5.3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d2e7225..a652efc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = 'dxfeed' -version = '0.5.2' +version = '0.5.3' description = 'DXFeed Python API via C API' authors = ['Index Management Team '] build = 'build.py' From 898f2b265019b8805a9fc265d4ce914cd741c241 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Fri, 3 Sep 2021 14:53:38 +0300 Subject: [PATCH 32/35] [EN-3260] toml and pandas version --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a652efc..5061342 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ exclude = ['dxfeed/tmp*'] [tool.poetry.dependencies] python = '^3.6' -pandas = [{version = '^1.0.0', python = '^3.6.1'}, {version = '^1.3.2', python = '^3.7.1'}] +pandas = [{version = '>=1.0,!=1.1.5,<1.2', python = '<3.7'}, {version = '^1.0.0', python = '^3.7'] toml = '^0.10.2' # The following dependencies are used for docs generation when installed as extras # (e.g. pip install nornir[docs]) @@ -46,7 +46,6 @@ pytest = '^6.2.4' sphinx = '^4.1.2' sphinx_rtd_theme = '^0.5.2' pygments = '^2.10' -toml = '^0.10.2' [tool.poetry.extras] docs = ['jupyter', 'cython'] From f90440f1e294e1007234c9d9bc1e9bcc2a579d8f Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Fri, 3 Sep 2021 15:17:11 +0300 Subject: [PATCH 33/35] [EN-3260] pandas version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5061342..68d6667 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ exclude = ['dxfeed/tmp*'] [tool.poetry.dependencies] python = '^3.6' -pandas = [{version = '>=1.0,!=1.1.5,<1.2', python = '<3.7'}, {version = '^1.0.0', python = '^3.7'] +pandas = {version = '^1.0.0', python = '^3.7'} toml = '^0.10.2' # The following dependencies are used for docs generation when installed as extras # (e.g. pip install nornir[docs]) From 82746a12fac846e93971b1a62be1dd6fa4813d9c Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Fri, 3 Sep 2021 15:25:25 +0300 Subject: [PATCH 34/35] [EN-3260] pandas version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 68d6667..f5f257e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ exclude = ['dxfeed/tmp*'] [tool.poetry.dependencies] python = '^3.6' -pandas = {version = '^1.0.0', python = '^3.7'} +pandas = '^1.0.0' toml = '^0.10.2' # The following dependencies are used for docs generation when installed as extras # (e.g. pip install nornir[docs]) From 0610ececde2e5b8a2d19c7e0734dd2623c7be936 Mon Sep 17 00:00:00 2001 From: Anatoly Kalin Date: Fri, 3 Sep 2021 15:31:44 +0300 Subject: [PATCH 35/35] [EN-3260] pandas version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f5f257e..5abe2de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ exclude = ['dxfeed/tmp*'] [tool.poetry.dependencies] python = '^3.6' -pandas = '^1.0.0' +pandas = [{version = '>=1.0,!=1.1.5,<1.2', python = '>=3.6.1,<3.7'}, {version = '^1.0.0', python = '^3.7'}] toml = '^0.10.2' # The following dependencies are used for docs generation when installed as extras # (e.g. pip install nornir[docs])