Skip to content

Commit

Permalink
rgw/s3select: json output format for csv, json & parquet
Browse files Browse the repository at this point in the history 
Signed-off-by: Albin Antony <[email protected]>
  • Loading branch information
@albin-antony
albin-antony committed Feb 13, 2024
1 parent e29d624 commit 9d6af2f
Showing 1 changed file with 20 additions and 35 deletions.
55 changes: 20 additions & 35 deletions s3tests_boto3/functional/test_s3select.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,7 @@ def test_count_operation():
obj_to_load = create_random_csv_object(num_of_rows,10)
upload_object(bucket_name,csv_obj_name,obj_to_load)
res = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object;") ).replace(",","")

s3select_assert_result( num_of_rows, int( res ))

@pytest.mark.s3select
Expand All @@ -407,16 +407,17 @@ def test_count_json_operation():
num_of_rows = 1
obj_to_load = create_random_json_object(num_of_rows,10)
upload_object(bucket_name,json_obj_name,obj_to_load)
res = remove_xml_tags_from_result(run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*];"))
s3select_assert_result( 1, int(res))
res = run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*];")
s3select_assert_result( '{"_1":1}\n', res)

res = remove_xml_tags_from_result(run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*].root;"))
s3select_assert_result( 1, int(res))
res = run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*].root;")
s3select_assert_result( '{"_1":1}\n', res)

json_obj_name = get_random_string()
obj_to_load = create_random_json_object(3,10)
upload_object(bucket_name,json_obj_name,obj_to_load)
res = remove_xml_tags_from_result(run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*].root;"))
s3select_assert_result( 3, int(res))
res = run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*].root;")
s3select_assert_result( '{"_1":3}\n', res)

@pytest.mark.s3select
def test_json_column_sum_min_max():
Expand All @@ -433,58 +434,42 @@ def test_json_column_sum_min_max():
bucket_name_2 = "testbuck2"
upload_object(bucket_name_2,json_obj_name_2,json_obj)

res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select min(_1.c1) from s3object[*].root;") ).replace(",","")
res_s3select = run_s3select_json(bucket_name,json_obj_name,"select min(_1.c1) from s3object[*].root;")
list_int = create_list_of_int( 1 , csv_obj )
res_target = min( list_int )

s3select_assert_result( int(res_s3select), int(res_target))
s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))

res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select min(_1.c4) from s3object[*].root;") ).replace(",","")
res_s3select = run_s3select_json(bucket_name,json_obj_name,"select min(_1.c4) from s3object[*].root;")
list_int = create_list_of_int( 4 , csv_obj )
res_target = min( list_int )

s3select_assert_result( int(res_s3select), int(res_target))

res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select avg(_1.c6) from s3object[*].root;") ).replace(",","")
list_int = create_list_of_int( 6 , csv_obj )
res_target = float(sum(list_int ))/10000

s3select_assert_result( float(res_s3select), float(res_target))
s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))

res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select max(_1.c4) from s3object[*].root;") ).replace(",","")
res_s3select = run_s3select_json(bucket_name,json_obj_name,"select max(_1.c4) from s3object[*].root;")
list_int = create_list_of_int( 4 , csv_obj )
res_target = max( list_int )

s3select_assert_result( int(res_s3select), int(res_target))
s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))

res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select max(_1.c7) from s3object[*].root;") ).replace(",","")
res_s3select = run_s3select_json(bucket_name,json_obj_name,"select max(_1.c7) from s3object[*].root;")
list_int = create_list_of_int( 7 , csv_obj )
res_target = max( list_int )

s3select_assert_result( int(res_s3select), int(res_target))
s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))

res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select sum(_1.c4) from s3object[*].root;") ).replace(",","")
res_s3select = run_s3select_json(bucket_name,json_obj_name,"select sum(_1.c4) from s3object[*].root;")
list_int = create_list_of_int( 4 , csv_obj )
res_target = sum( list_int )

s3select_assert_result( int(res_s3select), int(res_target))
s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))

res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select sum(_1.c7) from s3object[*].root;") ).replace(",","")
res_s3select = run_s3select_json(bucket_name,json_obj_name,"select sum(_1.c7) from s3object[*].root;")
list_int = create_list_of_int( 7 , csv_obj )
res_target = sum( list_int )

s3select_assert_result( int(res_s3select) , int(res_target) )

# the following queries, validates on *random* input an *accurate* relation between condition result,sum operation and count operation.
res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name_2,json_obj_name_2,"select count(0),sum(_1.c1),sum(_1.c2) from s3object[*].root where (_1.c1-_1.c2) = 2;" ) )
count,sum1,sum2 = res_s3select.split(",")
s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))

s3select_assert_result( int(count)*2 , int(sum1)-int(sum2 ) )

res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select count(0),sum(_1.c1),sum(_1.c2) from s3object[*].root where (_1.c1-_1.c2) = 4;" ) )
count,sum1,sum2 = res_s3select.split(",")

s3select_assert_result( int(count)*4 , int(sum1)-int(sum2) )

@pytest.mark.s3select
def test_json_nullif_expressions():
Expand Down

0 comments on commit 9d6af2f

Please sign in to comment.