Skip to content

Commit

Permalink
feat: Add Python 3.11.4
Browse files Browse the repository at this point in the history
  • Loading branch information
assambar committed Jun 23, 2023
1 parent 7be7939 commit d1d83f1
Show file tree
Hide file tree
Showing 11 changed files with 345 additions and 18 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/build-python.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,16 @@ jobs:
include:
- prefix: ""
suffix: ""
version: "3.11.3"
version: "3.11.4"
- prefix: "wasmedge-"
suffix: "-wasmedge"
version: "3.11.3"
version: "3.11.4"
- prefix: "aio-"
suffix: "-aio"
version: "3.11.3"
version: "3.11.4"
- prefix: "aio-wasmedge-"
suffix: "-aio-wasmedge"
version: "3.11.3"
version: "3.11.4"
runs-on: ubuntu-latest
steps:
- name: Checkout repository
Expand Down
28 changes: 14 additions & 14 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -67,42 +67,42 @@ oci-python-3.11.1-wasmedge: python/wasmedge-v3.11.1
-f images/python/Dockerfile \
build-output

.PHONY: python/wasmedge-v3.11.3
python/wasmedge-v3.11.3:
.PHONY: python/wasmedge-v3.11.4
python/wasmedge-v3.11.4:
WLR_BUILD_FLAVOR=wasmedge \
make -C python $(subst python/wasmedge-,,$@)

.PHONY: python/aio-v3.11.3
python/aio-v3.11.3:
.PHONY: python/aio-v3.11.4
python/aio-v3.11.4:
WLR_BUILD_FLAVOR=aio \
make -C python $(subst python/aio-,,$@)

.PHONY: python/aio-wasmedge-v3.11.3
python/aio-wasmedge-v3.11.3:
.PHONY: python/aio-wasmedge-v3.11.4
python/aio-wasmedge-v3.11.4:
WLR_BUILD_FLAVOR=aio-wasmedge \
make -C python $(subst python/aio-wasmedge-,,$@)

.PHONY: oci-python-3.11.3
oci-python-3.11.3: python/v3.11.3
.PHONY: oci-python-3.11.4
oci-python-3.11.4: python/v3.11.4
docker build \
--platform wasi/wasm32 \
--build-arg NAME=python-wasm \
--build-arg SUMMARY="CPython built for WASI, by Wasm Labs" \
--build-arg ARTIFACTS_BASE_DIR=python/v3.11.3 \
--build-arg ARTIFACTS_BASE_DIR=python/v3.11.4 \
--build-arg PYTHON_BINARY=python.wasm \
-t ghcr.io/vmware-labs/python-wasm:3.11.3 \
-t ghcr.io/vmware-labs/python-wasm:3.11.4 \
-f images/python/Dockerfile \
build-output

.PHONY: oci-python-3.11.3-wasmedge
oci-python-3.11.3-wasmedge: python/wasmedge-v3.11.3
.PHONY: oci-python-3.11.4-wasmedge
oci-python-3.11.3-wasmedge: python/wasmedge-v3.11.4
docker build \
--platform wasi/wasm32 \
--build-arg NAME=python-wasm \
--build-arg SUMMARY="CPython built for WASI+WasmEdge, by Wasm Labs" \
--build-arg ARTIFACTS_BASE_DIR=python/v3.11.3-wasmedge \
--build-arg ARTIFACTS_BASE_DIR=python/v3.11.4-wasmedge \
--build-arg PYTHON_BINARY=python.wasm \
-t ghcr.io/vmware-labs/python-wasm:3.11.3-wasmedge \
-t ghcr.io/vmware-labs/python-wasm:3.11.4-wasmedge \
-f images/python/Dockerfile \
build-output

Expand Down
28 changes: 28 additions & 0 deletions python/test/run_me.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/usr/bin/env bash

if [[ "$(realpath $PWD)" != "$(realpath $(dirname $BASH_SOURCE))" ]]
then
echo "This script works only if called from its location as PWD"
exit 1
fi

TESTS=$(for t in */test.py; do echo $(dirname $t); done)

status=0

for test_dir in "$TESTS"; do
echo "Running test in '${test_dir}'..."
if diff ${test_dir}/test.stdout <($WLR_TEST_RUNTIME \
--mapdir /usr::${WLR_OUTPUT}/usr \
--mapdir /test::./${test_dir} \
--mapdir .::./${test_dir} \
${WLR_TESTED_MODULE} \
/test/test.py); then
echo "${test_dir}/test.py passed!"
else
echo "${test_dir}/test.py failed!"
status=$(expr ${status} + 1)
fi
done

exit $status
1 change: 1 addition & 0 deletions python/test/sqlite/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
*.db
74 changes: 74 additions & 0 deletions python/test/sqlite/test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import traceback

import sqlite3
import sys

DB_FILE_NAME = 'test.db'

def create_table(con, name, schema):
cursorObj = con.cursor()
cursorObj.execute(f"CREATE TABLE {name}{schema}")
con.commit()

def populate_table(con, name):
cursorObj = con.cursor()
cursorObj.execute(f"INSERT INTO {name} VALUES(1, 'John', 700, 'TheDarkMaster')")
cursorObj.execute(f"INSERT INTO {name} VALUES(2, 'Jane', 710, 'AngelOfBenevolence')")
cursorObj.execute(f"INSERT INTO {name} VALUES(3, 'George', 623, 'MagiFromTheDeep')")
con.commit()

def select_table(con, name):
cursorObj = con.cursor()
cursorObj.execute(f"SELECT * FROM {name}")
rows = cursorObj.fetchall()
for row in rows:
print(row)

def sql_version(con):
cursor = con.cursor()

sqlite_select_Query = "select sqlite_version();"
cursor.execute(sqlite_select_Query)
record = cursor.fetchall()
print("SQLite Database Version is: ", record)
cursor.close()

connection = None
try:
connection = sqlite3.connect(DB_FILE_NAME)
print("Database created and Successfully Connected to SQLite")

sql_version(connection)

try:
create_table(connection, "players", "(id integer PRIMARY KEY, name text, score real, nickname text)")
except sqlite3.Error as e:
print("Error while trying to create table: ", e)

try:
populate_table(connection, "players")
except sqlite3.Error as e:
print("Error while trying to create table: ", e)


try:
select_table(connection, "players")

except sqlite3.Error as e:
print("Error while trying to fetch table: ", e)


except sqlite3.Error as error:
print("Error while connecting to sqlite", error)

print(traceback.format_exc())

finally:
if connection:
connection.close()
print("The SQLite connection is closed")
import os
if os.path.isfile(DB_FILE_NAME):
os.remove(DB_FILE_NAME)
print(f"File '{DB_FILE_NAME}' deleted!")

7 changes: 7 additions & 0 deletions python/test/sqlite/test.stdout
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Database created and Successfully Connected to SQLite
SQLite Database Version is: [('3.42.0',)]
(1, 'John', 700.0, 'TheDarkMaster')
(2, 'Jane', 710.0, 'AngelOfBenevolence')
(3, 'George', 623.0, 'MagiFromTheDeep')
The SQLite connection is closed
File 'test.db' deleted!
132 changes: 132 additions & 0 deletions python/v3.11.4/wlr-build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
#!/usr/bin/env bash

if [[ ! -v WLR_ENV ]]
then
echo "WLR build environment is not set"
exit 1
fi

cd "${WLR_SOURCE_PATH}"

if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]]
then
source ${WLR_REPO_ROOT}/scripts/build-helpers/wlr_wasi_vfs.sh
wlr_wasi_vfs_setup_dependencies || exit 1
fi

source ${WLR_REPO_ROOT}/scripts/build-helpers/wlr_pkg_config.sh

export CFLAGS_CONFIG="-O0"


# This fails with upgraded clang for wasi-sdk19 and later. Disabled on cpython main.
#
# PyModule_AddIntMacro(module, CLOCK_MONOTONIC) and the like cause this.
# In all POSIX variants CLOCK_MONOTONIC is a numeric constant, so python imports it as int macro
# However, in wasi-libc clockid_t is defined as a pointer to struct __clockid.

export CFLAGS_CONFIG="${CFLAGS_CONFIG} -Wno-int-conversion"

export CFLAGS="${CFLAGS_CONFIG} ${CFLAGS_DEPENDENCIES} ${CFLAGS}"
export LDFLAGS="${LDFLAGS_DEPENDENCIES} ${LDFLAGS}"

export PYTHON_WASM_CONFIGURE="--with-build-python=python3"

if [[ "${WLR_BUILD_FLAVOR}" == *"wasmedge"* ]]
then
if [[ ! -v WABT_ROOT ]]
then
echo "WABT_ROOT is needed to patch imports for wasmedge"
exit 1
fi
fi

# By exporting WLR_SKIP_WASM_OPT envvar during the build, the
# wasm-opt wrapper in the wasm-base image will be a dummy wrapper that
# is effectively a NOP.
#
# This is due to https://github.com/llvm/llvm-project/issues/55781, so
# that we get to choose which optimization passes are executed after
# the artifacts have been built.
export WLR_SKIP_WASM_OPT=1

if [[ -z "$WLR_SKIP_CONFIGURE" ]]; then
logStatus "Configuring build with '${PYTHON_WASM_CONFIGURE}'... "
CONFIG_SITE=./Tools/wasm/config.site-wasm32-wasi ./configure -C --host=wasm32-wasi --build=$(./config.guess) ${PYTHON_WASM_CONFIGURE} || exit 1
else
logStatus "Skipping configure..."
fi

export MAKE_TARGETS='python.wasm wasm_stdlib'

logStatus "Building '${MAKE_TARGETS}'... "
make -j ${MAKE_TARGETS} || exit 1

unset WLR_SKIP_WASM_OPT

if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]]
then
logStatus "Packing with wasi-vfs"
wlr_wasi_vfs_cli pack python.wasm --mapdir /usr::$PWD/usr -o python.wasm || exit 1
fi

logStatus "Optimizing python binary..."
wasm-opt -O2 -o python-optimized.wasm python.wasm || exit 1

if [[ "${WLR_BUILD_FLAVOR}" == *"wasmedge"* ]]
then
logStatus "Patching python binary for wasmedge..."
${WLR_REPO_ROOT}/scripts/build-helpers/patch_wasmedge_wat_sock_accept.sh python-optimized.wasm || exit 1
fi

logStatus "Preparing artifacts... "
TARGET_PYTHON_BINARY=${WLR_OUTPUT}/bin/python.wasm

mkdir -p ${WLR_OUTPUT}/bin 2>/dev/null || exit 1

if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]]
then
cp -v python-optimized.wasm ${TARGET_PYTHON_BINARY} || exit 1
else
mkdir -p ${WLR_OUTPUT}/usr 2>/dev/null || exit 1
cp -v python-optimized.wasm ${TARGET_PYTHON_BINARY} || exit 1
cp -TRv usr ${WLR_OUTPUT}/usr || exit 1
fi

if [[ "${WLR_BUILD_FLAVOR}" != *"aio"* && "${WLR_BUILD_FLAVOR}" != *"wasmedge"* ]]
then

logStatus "Install includes..."
make inclinstall \
prefix=${WLR_OUTPUT} \
libdir=${WLR_OUTPUT}/lib/wasm32-wasi \
pkgconfigdir=${WLR_OUTPUT}/lib/wasm32-wasi/pkgconfig || exit 1

logStatus "Create libpython3.11-aio.a"
(${AR} -M <<EOF
create libpython3.11-aio.a
addlib libpython3.11.a
addlib ${WLR_DEPS_ROOT}/build-output/lib/wasm32-wasi/libz.a
addlib ${WLR_DEPS_ROOT}/build-output/lib/wasm32-wasi/libbz2.a
addlib ${WLR_DEPS_ROOT}/build-output/lib/wasm32-wasi/libsqlite3.a
addlib ${WLR_DEPS_ROOT}/build-output/lib/wasm32-wasi/libuuid.a
addlib Modules/expat/libexpat.a
addlib Modules/_decimal/libmpdec/libmpdec.a
save
end
EOF
) || echo exit 1

mkdir -p ${WLR_OUTPUT}/lib/wasm32-wasi/ 2>/dev/null || exit 1
cp -v libpython3.11-aio.a ${WLR_OUTPUT}/lib/wasm32-wasi/libpython3.11.a || exit 1

logStatus "Generating pkg-config file for libpython3.11.a"
DESCRIPTION="libpython3.11 allows embedding the CPython interpreter"
EXTRA_LINK_FLAGS="-lpython3.11 -Wl,-z,stack-size=524288 -Wl,--stack-first -Wl,--initial-memory=10485760 -lwasi-emulated-getpid -lwasi-emulated-signal -lwasi-emulated-process-clocks"

PC_INCLUDE_SUBDIR=python3.11 wlr_pkg_config_create_pc_file "libpython3.11" "${WLR_PACKAGE_VERSION}" "${DESCRIPTION}" "${EXTRA_LINK_FLAGS}" || exit 1

WLR_PACKAGE_LIB_EXTRA_DIRS=usr wlr_package_lib || exit 1
fi

logStatus "DONE. Artifacts in ${WLR_OUTPUT}"
17 changes: 17 additions & 0 deletions python/v3.11.4/wlr-env-repo.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/usr/bin/env bash

if [[ $1 == "--unset" ]]
then
unset WLR_REPO
unset WLR_REPO_BRANCH
unset WLR_ENV_NAME
unset WLR_PACKAGE_VERSION
unset WLR_PACKAGE_NAME
return
fi

export WLR_REPO=https://github.com/python/cpython
export WLR_REPO_BRANCH=v3.11.4
export WLR_ENV_NAME=python/v3.11.4
export WLR_PACKAGE_VERSION=3.11.4
export WLR_PACKAGE_NAME=python
20 changes: 20 additions & 0 deletions python/v3.11.4/wlr-info.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"deps": {
"libuuid": {
"build_target": "libs/libuuid/v1.0.3",
"required_file": "lib/wasm32-wasi/libuuid.a"
},
"zlib": {
"build_target": "libs/zlib/v1.2.13",
"required_file": "lib/wasm32-wasi/libz.a"
},
"SQLite": {
"build_target": "libs/sqlite/v3.42.0",
"required_file": "lib/wasm32-wasi/libsqlite3.a"
},
"bzip2": {
"build_target": "libs/bzip2/v1.0.8",
"required_file": "lib/wasm32-wasi/libbz2.a"
}
}
}
1 change: 1 addition & 0 deletions python/v3.11.4/wlr-tag.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
export WLR_TAG="python/3.11.4"
Loading

0 comments on commit d1d83f1

Please sign in to comment.