From 147dfd6f5b41e8e25f7eb12659dd345865ca793b Mon Sep 17 00:00:00 2001 From: EscottC Date: Thu, 27 Jun 2019 14:08:22 -0700 Subject: [PATCH 01/26] added todo list --- awkward-cpp/awkward/cpp/array/jagged.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index dbcdbc7e..458f585b 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -1,5 +1,12 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * TODO: += Expand array types + - std::string + - py::object + - zero-terminated bytes + - raw data += Make strides work negatively += Add more getitem functions = Deal with more array characteristics - Multidimensional arrays * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ From 9c23a3c5d55f794a16ba2bdcf20c8bab94d19034 Mon Sep 17 00:00:00 2001 From: EscottC Date: Fri, 28 Jun 2019 18:41:53 -0700 Subject: [PATCH 02/26] slices --- awkward-cpp/awkward/cpp/array/any.h | 10 ++-- awkward-cpp/awkward/cpp/array/array_impl.cpp | 2 +- awkward-cpp/awkward/cpp/array/jagged.h | 62 +++++++++++++++----- awkward-cpp/awkward/cpp/array/jagged.py | 18 ++++++ awkward-cpp/awkward/cpp/array/numpytypes.h | 15 ++--- awkward-cpp/awkward/cpp/array/util.h | 59 ------------------- 6 files changed, 79 insertions(+), 87 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/any.h b/awkward-cpp/awkward/cpp/array/any.h index 57876b26..55e420ad 100644 --- a/awkward-cpp/awkward/cpp/array/any.h +++ b/awkward-cpp/awkward/cpp/array/any.h @@ -7,15 +7,15 @@ namespace py = pybind11; class AnyOutput { public: - virtual AnyOutput* getitem(ssize_t) = 0; - virtual py::object unwrap() = 0; - virtual std::string str() = 0; + virtual AnyOutput* getitem(ssize_t) = 0; + virtual py::object unwrap() = 0; + virtual std::string str() = 0; }; class AnyArray : public AnyOutput { public: - virtual ssize_t len() = 0; - virtual AnyArray* getitem(ssize_t, ssize_t) = 0; + virtual ssize_t len() = 0; + virtual AnyArray* getitem(ssize_t a, ssize_t b, ssize_t c = 1) = 0; }; class AwkwardArray : public AnyArray { diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index b499ada5..990760f5 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -12,7 +12,7 @@ PYBIND11_MODULE(array_impl, m) { .def_static("parents2startsstops", &JaggedArray::parents2startsstops) .def_static("uniques2offsetsparents", &JaggedArray::uniques2offsetsparents) .def("__getitem__", (py::object (JaggedArray::*)(ssize_t)) &JaggedArray::python_getitem) - .def("__getitem__", (py::object (JaggedArray::*)(ssize_t, ssize_t)) &JaggedArray::python_getitem) + .def("__getitem__", (py::object (JaggedArray::*)(ssize_t, ssize_t, ssize_t)) &JaggedArray::python_getitem) .def("__str__", &JaggedArray::str) .def("__len__", &JaggedArray::len) .def("__iter__", &JaggedArray::iter) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 458f585b..f9166e95 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -6,6 +6,10 @@ - zero-terminated bytes - raw data = Make strides work negatively += Add c++ indexing util function for slicing + - negative indices work like C, not python + - python-style can be implemented when called + - slices will have to be interpreted on python side = Add more getitem functions = Deal with more array characteristics - Multidimensional arrays @@ -322,41 +326,69 @@ class JaggedArray : public AwkwardArray { return starts.request().size; } - AnyArray* getitem(ssize_t start, ssize_t end) { - if (start < 0) { - start += len(); + AnyArray* getitem(ssize_t start, ssize_t length, ssize_t step = 1) { + if (step == 0) { + throw std::invalid_argument("slice step cannot be 0"); } - if (end < 0) { - end += len(); + if (length < 0) { + throw std::invalid_argument("slice length cannot be less than 0"); } - if (start < 0 || start > end || end > len()) { - throw std::out_of_range("getitem must be in the bounds of the array"); + if (start < 0 || start >= len() || start + (length * step) > len() || start + (length * step) < -1) { + throw std::out_of_range("getitem must be in the bounds of the array."); } - auto newStarts = py::array_t(end - start); + auto newStarts = py::array_t(length); py::buffer_info newStarts_info = newStarts.request(); auto newStarts_ptr = (std::int64_t*)newStarts_info.ptr; py::buffer_info starts_info = starts.request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; + int N_starts = starts_info.strides[0] / starts_info.itemsize; - auto newStops = py::array_t(end - start); + auto newStops = py::array_t(length); py::buffer_info newStops_info = newStops.request(); auto newStops_ptr = (std::int64_t*)newStops_info.ptr; py::buffer_info stops_info = stops.request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; + int N_stops = stops_info.strides[0] / stops_info.itemsize; ssize_t newIndex = 0; - for (ssize_t i = start; i < end; i++) { - newStarts_ptr[newIndex] = starts_ptr[i]; - newStops_ptr[newIndex++] = stops_ptr[i]; + for (ssize_t i = 0; i < length; i++) { + newStarts_ptr[newIndex] = starts_ptr[start + (i * step * N_starts)]; + newStops_ptr[newIndex++] = stops_ptr[start + (i * step * N_stops)]; } return new JaggedArray(newStarts, newStops, content); } - py::object python_getitem(ssize_t start, ssize_t end) { - return getitem(start, end)->unwrap(); + py::object python_getitem(ssize_t start, ssize_t stop, ssize_t step) { + if (step == 0) { + throw std::invalid_argument("slice step cannot be 0"); + } + ssize_t length = len(); + if (start < 0) { + start += length; + } + if (stop < 0) { + stop += length; + } + if (step > 0) { + if (stop > length) { + stop = length; + } + if (start >= stop) { + return getitem(length - 1, 0, step)->unwrap(); + } + } + else { + if (stop < -1) { + stop = -1; + } + if (start <= stop) { + return getitem(0, 0, step)->unwrap(); + } + } + return getitem(start, (stop + step - start) / step, step)->unwrap(); } AnyArray* getitem(ssize_t index) { @@ -377,7 +409,7 @@ class JaggedArray : public AwkwardArray { ssize_t start = (ssize_t)((std::int64_t*)starts_info.ptr)[index]; ssize_t stop = (ssize_t)((std::int64_t*)stops_info.ptr)[index]; - return content->getitem(start, stop); + return content->getitem(start, stop - start); } py::object python_getitem(ssize_t index) { diff --git a/awkward-cpp/awkward/cpp/array/jagged.py b/awkward-cpp/awkward/cpp/array/jagged.py index cdfce337..4c279ad2 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.py +++ b/awkward-cpp/awkward/cpp/array/jagged.py @@ -15,3 +15,21 @@ def parents2startsstops(cls, parents, length = None): if length is None: length = -1 return getattr(JaggedArray, "parents2startsstops")(parents, length) + + def __getitem__(self, where): + if isinstance(where, slice): + length = getattr(JaggedArray, "__len__")(self) + start = 0 + stop = length + step = 1 + if where.step is not None: + step = where.step + if step < 0: + start = length - 1 + stop = -1 - length + if where.start is not None: + start = where.start + if where.stop is not None: + stop = where.stop + return getattr(JaggedArray, "__getitem__")(self, start, stop, step) + return getattr(JaggedArray, "__getitem__")(self, where) diff --git a/awkward-cpp/awkward/cpp/array/numpytypes.h b/awkward-cpp/awkward/cpp/array/numpytypes.h index 8143e6df..74f89a88 100644 --- a/awkward-cpp/awkward/cpp/array/numpytypes.h +++ b/awkward-cpp/awkward/cpp/array/numpytypes.h @@ -55,23 +55,24 @@ class NumpyArray_t : public NumpyArray { return thisArray.request().size; } - AnyArray* getitem(ssize_t start, ssize_t end) { - if (start < 0) { - start += thisArray.request().size; + AnyArray* getitem(ssize_t start, ssize_t length, ssize_t step = 1) { + if (step == 0) { + throw std::invalid_argument("slice step cannot be 0"); } - if (end < 0) { - end += thisArray.request().size; + if (length < 0) { + throw std::invalid_argument("slice length cannot be less than 0"); } - if (start < 0 || start > end || end > thisArray.request().size) { + if (start < 0 || start >= len() || start + (length * step) > len() || start + (length * step) < -1) { throw std::out_of_range("getitem must be in the bounds of the array"); } py::buffer_info temp_info = py::buffer_info(); temp_info.ptr = (void*)((T*)(thisArray.request().ptr) + start); temp_info.itemsize = thisArray.request().itemsize; - temp_info.size = end - start; + temp_info.size = length; temp_info.format = thisArray.request().format; temp_info.ndim = thisArray.request().ndim; temp_info.strides = thisArray.request().strides; + temp_info.strides[0] = temp_info.strides[0] * step; temp_info.shape = thisArray.request().shape; temp_info.shape[0] = temp_info.size; return new NumpyArray_t(py::array(temp_info)); diff --git a/awkward-cpp/awkward/cpp/array/util.h b/awkward-cpp/awkward/cpp/array/util.h index 6a16cb05..304f64b2 100644 --- a/awkward-cpp/awkward/cpp/array/util.h +++ b/awkward-cpp/awkward/cpp/array/util.h @@ -111,62 +111,3 @@ void makeIntNative(py::array input) { } throw std::invalid_argument("argument must be of type int"); } - -std::string convert_to_string(std::int64_t val) { - return std::to_string(val); -} - -std::string convert_to_string(std::uint64_t val) { - return std::to_string(val); -} - -std::string convert_to_string(std::int32_t val) { - return std::to_string(val); -} - -std::string convert_to_string(std::uint32_t val) { - return std::to_string(val); -} - -std::string convert_to_string(std::int16_t val) { - return std::to_string(val); -} - -std::string convert_to_string(std::uint16_t val) { - return std::to_string(val); -} - -std::string convert_to_string(std::int8_t val) { - return std::to_string((std::int16_t)val); -} - -std::string convert_to_string(std::uint8_t val) { - return std::to_string((std::uint16_t)val); -} - -std::string trim_trail(std::string val) { - size_t end = val.find_last_not_of("0"); - if (end == std::string::npos) { - return ""; - } - else if (val.at(end) == '.') { - return val.substr(0, end + 1) + "0"; - } - return val.substr(0, end + 1); -} - -std::string convert_to_string(float val) { - return trim_trail(std::to_string(val)); -} - -std::string convert_to_string(double val) { - return trim_trail(std::to_string(val)); -} - -std::string convert_to_string(std::complex val) { - return "(" + trim_trail(std::to_string(real(val))) + " + " + trim_trail(std::to_string(imag(val))) + "i)"; -} - -std::string convert_to_string(std::complex val) { - return "(" + trim_trail(std::to_string(real(val))) + " + " + trim_trail(std::to_string(imag(val))) + "i)"; -} From 8ad1daf4a614013d5cb85b8d55807c2b848de139 Mon Sep 17 00:00:00 2001 From: EscottC Date: Mon, 1 Jul 2019 15:11:52 -0700 Subject: [PATCH 03/26] numpy array slicing, check validity, from offsets, from counts - Added a utility function in util.h to slice a py::array - Added a non-static function to check the starts/stops/content validity in jagged.h whenever a JaggedArray is instantiated - Added fromoffsets and python_fromoffsets - Added fromcounts and python_fromcounts --- awkward-cpp/awkward/cpp/array/array_impl.cpp | 2 + awkward-cpp/awkward/cpp/array/jagged.h | 92 ++++++++++++++++++-- awkward-cpp/awkward/cpp/array/numpytypes.h | 21 +---- awkward-cpp/awkward/cpp/array/util.h | 20 +++++ 4 files changed, 110 insertions(+), 25 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index 990760f5..01ea49ac 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -11,6 +11,8 @@ PYBIND11_MODULE(array_impl, m) { .def_static("startsstops2parents", &JaggedArray::startsstops2parents) .def_static("parents2startsstops", &JaggedArray::parents2startsstops) .def_static("uniques2offsetsparents", &JaggedArray::uniques2offsetsparents) + .def_static("fromoffsets", &JaggedArray::python_fromoffsets) + .def_static("fromcounts", &JaggedArray::python_fromcounts) .def("__getitem__", (py::object (JaggedArray::*)(ssize_t)) &JaggedArray::python_getitem) .def("__getitem__", (py::object (JaggedArray::*)(ssize_t, ssize_t, ssize_t)) &JaggedArray::python_getitem) .def("__str__", &JaggedArray::str) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index f9166e95..2981ebfe 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -5,11 +5,6 @@ - py::object - zero-terminated bytes - raw data -= Make strides work negatively -= Add c++ indexing util function for slicing - - negative indices work like C, not python - - python-style can be implemented when called - - slices will have to be interpreted on python side = Add more getitem functions = Deal with more array characteristics - Multidimensional arrays @@ -99,18 +94,105 @@ class JaggedArray : public AwkwardArray { stops = stops_; } + bool check_validity() { + py::buffer_info starts_info = starts.request(); + py::buffer_info stops_info = stops.request(); + if (starts_info.size > stops_info.size) { + throw std::invalid_argument("starts must have the same (or shorter) length than stops"); + } + if (starts_info.ndim != stops_info.ndim) { + throw std::domain_error("starts and stops must have the same dimensionality"); + } + int N_starts = starts_info.strides[0] / starts_info.itemsize; + int N_stops = stops_info.strides[0] / stops_info.itemsize; + std::int64_t starts_max = 0; + std::int64_t stops_max = 0; + auto starts_ptr = (std::int64_t*)starts_info.ptr; + auto stops_ptr = (std::int64_t*)stops_info.ptr; + for (ssize_t i = 0; i < starts_info.size; i++) { + if (stops_ptr[i * N_stops] < starts_ptr[i * N_starts]) { + throw std::invalid_argument("stops must be greater than or equal to starts"); + } + if (starts_ptr[i * N_starts] > starts_max) { + starts_max = starts_ptr[i * N_starts]; + } + if (stops_ptr[i * N_stops] > stops_max) { + stops_max = stops_ptr[i * N_stops]; + } + } + if (starts_info.size > 0) { + if (starts_max >= content->len()) { + throw std::invalid_argument("The maximum of starts for non-empty elements must be less than the length of content"); + } + if (stops_max > content->len()) { + throw std::invalid_argument("The maximum of stops for non-empty elements must be less than or equal to the length of content"); + } + } + return true; + } + JaggedArray(py::array starts_, py::array stops_, py::object content_) { set_starts(starts_); set_stops(stops_); python_set_content(content_); + check_validity(); } JaggedArray(py::array starts_, py::array stops_, AnyArray* content_) { set_starts(starts_); set_stops(stops_); set_content(content_); + check_validity(); + } + + static JaggedArray* fromoffsets(py::array offsets, AnyArray* content_) { + makeIntNative(offsets); + py::array_t temp = offsets.cast>(); + ssize_t length = temp.request().size; + if (length < 1) { + throw std::invalid_argument("offsets must have at least one element"); + } + if (temp.request().ndim > 1) { + throw std::domain_error("offsets must be one-dimensional"); + } + return new JaggedArray( + slice_numpy(temp, 0, length - 1), + slice_numpy(temp, 1, length - 1), + content_ + ); } + static JaggedArray* python_fromoffsets(py::array offsets, py::object content_) { + try { + return fromoffsets(offsets, content_.cast()); + } + catch (py::cast_error e) { } + try { + return fromoffsets(offsets, getNumpyArray_t(content_.cast())); + } + catch (py::cast_error e) { + throw std::invalid_argument("Invalid type for JaggedArray.content"); + } + } + + static JaggedArray* fromcounts(py::array counts, AnyArray* content_) { + return fromoffsets(counts2offsets(counts), content_); + } + + static JaggedArray* python_fromcounts(py::array counts, py::object content_) { + try { + return fromcounts(counts, content_.cast()); + } + catch (py::cast_error e) { } + try { + return fromcounts(counts, getNumpyArray_t(content_.cast())); + } + catch (py::cast_error e) { + throw std::invalid_argument("Invalid type for JaggedArray.content"); + } + } + + static py::array_t offsets2parents(py::array offsets) { makeIntNative(offsets); offsets = offsets.cast>(); diff --git a/awkward-cpp/awkward/cpp/array/numpytypes.h b/awkward-cpp/awkward/cpp/array/numpytypes.h index 74f89a88..53cdd7ad 100644 --- a/awkward-cpp/awkward/cpp/array/numpytypes.h +++ b/awkward-cpp/awkward/cpp/array/numpytypes.h @@ -56,26 +56,7 @@ class NumpyArray_t : public NumpyArray { } AnyArray* getitem(ssize_t start, ssize_t length, ssize_t step = 1) { - if (step == 0) { - throw std::invalid_argument("slice step cannot be 0"); - } - if (length < 0) { - throw std::invalid_argument("slice length cannot be less than 0"); - } - if (start < 0 || start >= len() || start + (length * step) > len() || start + (length * step) < -1) { - throw std::out_of_range("getitem must be in the bounds of the array"); - } - py::buffer_info temp_info = py::buffer_info(); - temp_info.ptr = (void*)((T*)(thisArray.request().ptr) + start); - temp_info.itemsize = thisArray.request().itemsize; - temp_info.size = length; - temp_info.format = thisArray.request().format; - temp_info.ndim = thisArray.request().ndim; - temp_info.strides = thisArray.request().strides; - temp_info.strides[0] = temp_info.strides[0] * step; - temp_info.shape = thisArray.request().shape; - temp_info.shape[0] = temp_info.size; - return new NumpyArray_t(py::array(temp_info)); + return new NumpyArray_t(slice_numpy(thisArray, start, length, step)); } AnyOutput* getitem(ssize_t i) { diff --git a/awkward-cpp/awkward/cpp/array/util.h b/awkward-cpp/awkward/cpp/array/util.h index 304f64b2..67be35cc 100644 --- a/awkward-cpp/awkward/cpp/array/util.h +++ b/awkward-cpp/awkward/cpp/array/util.h @@ -111,3 +111,23 @@ void makeIntNative(py::array input) { } throw std::invalid_argument("argument must be of type int"); } + +template +py::array_t slice_numpy(py::array_t input, ssize_t start, ssize_t length, ssize_t step = 1) { + ssize_t arrayLen = input.request().size; + if (step == 0) { + throw std::invalid_argument("slice step cannot be 0"); + } + if (length < 0) { + throw std::invalid_argument("slice length cannot be less than 0"); + } + if (start < 0 || start >= arrayLen || start + (length * step) > arrayLen || start + (length * step) < -1) { + throw std::out_of_range("slice must be in the bounds of the array"); + } + py::buffer_info temp_info = py::buffer_info(input.request()); + temp_info.ptr = (void*)((T*)(input.request().ptr) + start); + temp_info.size = length; + temp_info.strides[0] = temp_info.strides[0] * step; + temp_info.shape[0] = temp_info.size; + return py::array_t(temp_info); +} From e6c92c53a945bedb1aad3c8e3a6d4a3324e0af96 Mon Sep 17 00:00:00 2001 From: EscottC Date: Mon, 1 Jul 2019 17:54:13 -0700 Subject: [PATCH 04/26] fromparents, fromuniques, fromjagged, copy, deepcopy - Added fromparents and python_fromparents - Added fromuniques and python_fromuniques - Added copy - Added deepcopy in JaggedArray and NumpyArray, pyarray_deepcopy in util.h, and python_deepcopy --- awkward-cpp/awkward/cpp/array/any.h | 1 + awkward-cpp/awkward/cpp/array/array_impl.cpp | 5 ++ awkward-cpp/awkward/cpp/array/jagged.h | 61 ++++++++++++++++++++ awkward-cpp/awkward/cpp/array/numpytypes.h | 4 ++ awkward-cpp/awkward/cpp/array/util.h | 12 ++++ 5 files changed, 83 insertions(+) diff --git a/awkward-cpp/awkward/cpp/array/any.h b/awkward-cpp/awkward/cpp/array/any.h index 55e420ad..edfb39da 100644 --- a/awkward-cpp/awkward/cpp/array/any.h +++ b/awkward-cpp/awkward/cpp/array/any.h @@ -16,6 +16,7 @@ class AnyArray : public AnyOutput { public: virtual ssize_t len() = 0; virtual AnyArray* getitem(ssize_t a, ssize_t b, ssize_t c = 1) = 0; + virtual AnyArray* deepcopy() = 0; }; class AwkwardArray : public AnyArray { diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index 01ea49ac..fe2e76f4 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -13,6 +13,11 @@ PYBIND11_MODULE(array_impl, m) { .def_static("uniques2offsetsparents", &JaggedArray::uniques2offsetsparents) .def_static("fromoffsets", &JaggedArray::python_fromoffsets) .def_static("fromcounts", &JaggedArray::python_fromcounts) + .def_static("fromparents", &JaggedArray::python_fromparents) + .def_static("fromuniques", &JaggedArray::python_fromuniques) + .def_static("fromjagged", &JaggedArray::fromjagged) + .def("copy", &JaggedArray::copy) + .def("deepcopy", &JaggedArray::deepcopy) .def("__getitem__", (py::object (JaggedArray::*)(ssize_t)) &JaggedArray::python_getitem) .def("__getitem__", (py::object (JaggedArray::*)(ssize_t, ssize_t, ssize_t)) &JaggedArray::python_getitem) .def("__str__", &JaggedArray::str) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 2981ebfe..23ada4e4 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -192,6 +192,67 @@ class JaggedArray : public AwkwardArray { } } + static JaggedArray* fromparents(py::array parents, AnyArray* content_, ssize_t length = -1) { + if (parents.request().ndim != 1 || parents.request().size != content_->len()) { + throw std::invalid_argument("parents array must be one-dimensional with the same length as content"); + } + auto startsstops = parents2startsstops(parents, length); + return new JaggedArray(startsstops[0], startsstops[1], content_); + } + + static JaggedArray* python_fromparents(py::array parents, py::object content_, ssize_t length = -1) { + try { + return fromparents(parents, content_.cast(), length); + } + catch (py::cast_error e) { } + try { + return fromparents(parents, getNumpyArray_t(content_.cast()), length); + } + catch (py::cast_error e) { + throw std::invalid_argument("Invalid type for JaggedArray.content"); + } + } + + static JaggedArray* fromuniques(py::array uniques, AnyArray* content_) { + if (uniques.request().ndim != 1 || uniques.request().size != content_->len()) { + throw std::invalid_argument("uniques array must be one-dimensional with the same length as content"); + } + auto offsetsparents = uniques2offsetsparents(uniques); + return fromoffsets(offsetsparents[0], content_); + } + + static JaggedArray* python_fromuniques(py::array uniques, py::object content_) { + try { + return fromuniques(uniques, content_.cast()); + } + catch (py::cast_error e) { } + try { + return fromuniques(uniques, getNumpyArray_t(content_.cast())); + } + catch (py::cast_error e) { + throw std::invalid_argument("Invalid type for JaggedArray.content"); + } + } + + static JaggedArray* fromjagged(JaggedArray* jagged) { + return new JaggedArray(jagged->get_starts(), jagged->get_stops(), jagged->get_content()); + } + + JaggedArray* copy() { + return new JaggedArray(starts, stops, content); + } + + AnyArray* deepcopy() { + return new JaggedArray( + pyarray_deepcopy(starts), + pyarray_deepcopy(stops), + content->deepcopy() + ); + } + + JaggedArray* python_deepcopy() { + return (JaggedArray*)deepcopy(); + } static py::array_t offsets2parents(py::array offsets) { makeIntNative(offsets); diff --git a/awkward-cpp/awkward/cpp/array/numpytypes.h b/awkward-cpp/awkward/cpp/array/numpytypes.h index 53cdd7ad..1d886082 100644 --- a/awkward-cpp/awkward/cpp/array/numpytypes.h +++ b/awkward-cpp/awkward/cpp/array/numpytypes.h @@ -51,6 +51,10 @@ class NumpyArray_t : public NumpyArray { return py::str(thisArray); } + AnyArray* deepcopy() { + return new NumpyArray_t(pyarray_deepcopy(thisArray)); + } + ssize_t len() { return thisArray.request().size; } diff --git a/awkward-cpp/awkward/cpp/array/util.h b/awkward-cpp/awkward/cpp/array/util.h index 67be35cc..0c9b4c94 100644 --- a/awkward-cpp/awkward/cpp/array/util.h +++ b/awkward-cpp/awkward/cpp/array/util.h @@ -131,3 +131,15 @@ py::array_t slice_numpy(py::array_t input, ssize_t start, ssize_t length, temp_info.shape[0] = temp_info.size; return py::array_t(temp_info); } + +template +py::array_t pyarray_deepcopy(py::array_t input) { + auto newArray = py::array_t(input.request().size); + auto newArray_ptr = (T*)newArray.request().ptr; + auto input_ptr = (T*)input.request().ptr; + int N = input.request().strides[0] / input.request().itemsize; + for (ssize_t i = 0; i < input.request().size; i++) { + newArray_ptr[i] = input_ptr[i * N]; + } + return newArray; +} From 8cf211415394a08dd09fc0b4d5480d224369e12a Mon Sep 17 00:00:00 2001 From: EscottC Date: Mon, 1 Jul 2019 18:24:14 -0700 Subject: [PATCH 05/26] small fixes - Fixed issue where one of the negative tests didn't catch an exception fast enough (because of the constructor validity check) - Moved python-esque getitem(ssize_t) negative index handling to the python-wrapping version of getitem(ssize_t) --- awkward-cpp/awkward/cpp/array/jagged.h | 6 +++--- tests/test_cpp.py | 18 ++++++++---------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 23ada4e4..8e3d8434 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -537,9 +537,6 @@ class JaggedArray : public AwkwardArray { AnyArray* getitem(ssize_t index) { py::buffer_info starts_info = starts.request(); py::buffer_info stops_info = stops.request(); - if (index < 0) { - index = starts_info.size + index; - } if (starts_info.size > stops_info.size) { throw std::out_of_range("starts must have the same or shorter length than stops"); } @@ -556,6 +553,9 @@ class JaggedArray : public AwkwardArray { } py::object python_getitem(ssize_t index) { + if (index < 0) { + index += starts.request().size; + } return getitem(index)->unwrap(); } diff --git a/tests/test_cpp.py b/tests/test_cpp.py index 33cf38e1..5a21ec1f 100644 --- a/tests/test_cpp.py +++ b/tests/test_cpp.py @@ -92,30 +92,28 @@ def test_cpp_init(self): test = awkward.cpp.JaggedArray(a, b, c) assert str(test) == "[[1] [4 5 6 7 8] [1 2 3 4] [0 1 2 3 4 5 6 7 8 9]]" - def test_cpp_getitem_neg01(self): + def test_cpp_init_neg01(self): a = numpy.array([1, 2, 3, 4]) b = numpy.array([2, 3, 4]) c = numpy.arange(10) - test = awkward.cpp.JaggedArray(a, b, c) thrown = False try: - d = test[0] - except IndexError as e: - if str(e) != "starts must have the same or shorter length than stops": + test = awkward.cpp.JaggedArray(a, b, c) + except ValueError as e: + if str(e) != "starts must have the same (or shorter) length than stops": raise thrown = True assert thrown - def test_cpp_getitem_neg02(self): + def test_cpp_init_neg02(self): a = numpy.array([0, 1, 2]) b = numpy.array([2, 3, 4]) c = numpy.arange(2) - test = awkward.cpp.JaggedArray(a, b, c) thrown = False try: - d = test[2] - except IndexError as e: - if str(e) != "getitem must be in the bounds of the array": + test = awkward.cpp.JaggedArray(a, b, c) + except ValueError as e: + if str(e) != "The maximum of starts for non-empty elements must be less than the length of content": raise thrown = True assert thrown From 459ccdc01bd68851656208137a4c3f16d8b417ba Mon Sep 17 00:00:00 2001 From: EscottC Date: Tue, 2 Jul 2019 13:01:58 -0700 Subject: [PATCH 06/26] tolist() --- awkward-cpp/awkward/cpp/array/any.h | 1 + awkward-cpp/awkward/cpp/array/array_impl.cpp | 1 + awkward-cpp/awkward/cpp/array/jagged.h | 14 ++++++++++++-- awkward-cpp/awkward/cpp/array/numpytypes.h | 10 ++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/any.h b/awkward-cpp/awkward/cpp/array/any.h index edfb39da..55256d86 100644 --- a/awkward-cpp/awkward/cpp/array/any.h +++ b/awkward-cpp/awkward/cpp/array/any.h @@ -10,6 +10,7 @@ class AnyOutput { virtual AnyOutput* getitem(ssize_t) = 0; virtual py::object unwrap() = 0; virtual std::string str() = 0; + virtual py::object tolist() = 0; }; class AnyArray : public AnyOutput { diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index fe2e76f4..1f7a2a66 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -18,6 +18,7 @@ PYBIND11_MODULE(array_impl, m) { .def_static("fromjagged", &JaggedArray::fromjagged) .def("copy", &JaggedArray::copy) .def("deepcopy", &JaggedArray::deepcopy) + .def("tolist", &JaggedArray::tolist) .def("__getitem__", (py::object (JaggedArray::*)(ssize_t)) &JaggedArray::python_getitem) .def("__getitem__", (py::object (JaggedArray::*)(ssize_t, ssize_t, ssize_t)) &JaggedArray::python_getitem) .def("__str__", &JaggedArray::str) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 8e3d8434..1cf24f12 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -546,8 +546,10 @@ class JaggedArray : public AwkwardArray { if (starts_info.ndim != stops_info.ndim) { throw std::domain_error("starts and stops must have the same dimensionality"); } - ssize_t start = (ssize_t)((std::int64_t*)starts_info.ptr)[index]; - ssize_t stop = (ssize_t)((std::int64_t*)stops_info.ptr)[index]; + int N_starts = starts_info.strides[0] / starts_info.itemsize; + int N_stops = stops_info.strides[0] / stops_info.itemsize; + ssize_t start = (ssize_t)((std::int64_t*)starts_info.ptr)[index * N_starts]; + ssize_t stop = (ssize_t)((std::int64_t*)stops_info.ptr)[index * N_stops]; return content->getitem(start, stop - start); } @@ -559,6 +561,14 @@ class JaggedArray : public AwkwardArray { return getitem(index)->unwrap(); } + py::object tolist() { + py::list out; + for (ssize_t i = 0; i < len(); i++) { + out.append(getitem(i)->tolist()); + } + return out; + } + std::string str() { std::string out; diff --git a/awkward-cpp/awkward/cpp/array/numpytypes.h b/awkward-cpp/awkward/cpp/array/numpytypes.h index 1d886082..fb7e2cd8 100644 --- a/awkward-cpp/awkward/cpp/array/numpytypes.h +++ b/awkward-cpp/awkward/cpp/array/numpytypes.h @@ -29,6 +29,8 @@ class NumpyScalar_t : public NumpyScalar { py::object unwrap() { return py::cast(thisScalar); } + py::object tolist() { return unwrap(); } + std::string str() { return py::str(unwrap()); } @@ -67,6 +69,14 @@ class NumpyArray_t : public NumpyArray { return new NumpyScalar_t(((T*)thisArray.request().ptr)[i]); } + py::object tolist() { + py::list out; + for (ssize_t i = 0; i < len(); i++) { + out.append(getitem(i)->tolist()); + } + return out; + } + NumpyArray_t(py::array_t input) { thisArray = input; } py::buffer_info request() { return thisArray.request(); } From 8e89a4e3b549b5af7ed4c902396cc7fbb322ccff Mon Sep 17 00:00:00 2001 From: EscottC Date: Tue, 2 Jul 2019 16:45:09 -0700 Subject: [PATCH 07/26] fromiter() --- awkward-cpp/awkward/cpp/array/array_impl.cpp | 1 + awkward-cpp/awkward/cpp/array/jagged.h | 41 ++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index 1f7a2a66..5ab2847a 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -11,6 +11,7 @@ PYBIND11_MODULE(array_impl, m) { .def_static("startsstops2parents", &JaggedArray::startsstops2parents) .def_static("parents2startsstops", &JaggedArray::parents2startsstops) .def_static("uniques2offsetsparents", &JaggedArray::uniques2offsetsparents) + .def_static("fromiter", &JaggedArray::fromiter) .def_static("fromoffsets", &JaggedArray::python_fromoffsets) .def_static("fromcounts", &JaggedArray::python_fromcounts) .def_static("fromparents", &JaggedArray::python_fromparents) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 1cf24f12..60f3d921 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -192,6 +192,47 @@ class JaggedArray : public AwkwardArray { } } + static AnyArray* fromiter_helper(py::tuple input) { + if (input.size() == 0) { + return getNumpyArray_t(py::array_t(0)); + } + try { + input[0].cast(); + return fromiter(input); + } + catch (std::exception e) { + py::array out = input.cast(); + return getNumpyArray_t(out); + } + } + + static JaggedArray* fromiter(py::object input) { + py::tuple iter = input.cast(); + auto counts = py::array_t(iter.size()); + auto counts_ptr = (std::int64_t*)counts.request().ptr; + + py::list contentList; + + if (iter.size() == 0) { + return fromcounts(counts, getNumpyArray_t(py::array_t(0))); + } + for (size_t i = 0; i < iter.size(); i++) { + py::tuple thisIter; + try { + thisIter = iter[i].cast(); + } + catch (std::exception e) { + throw std::invalid_argument("jagged iterable must contain only iterables to make a jagged array"); + } + counts_ptr[i] = (std::int64_t)thisIter.size(); + for (size_t i = 0; i < thisIter.size(); i++) { + contentList.append(thisIter[i]); + } + } + auto content_out = py::tuple(contentList); + return fromcounts(counts, fromiter_helper(content_out)); + } + static JaggedArray* fromparents(py::array parents, AnyArray* content_, ssize_t length = -1) { if (parents.request().ndim != 1 || parents.request().size != content_->len()) { throw std::invalid_argument("parents array must be one-dimensional with the same length as content"); From 0290783f9e7b67dbc4f6b40470db26e4a36ad059 Mon Sep 17 00:00:00 2001 From: EscottC Date: Tue, 2 Jul 2019 19:07:17 -0700 Subject: [PATCH 08/26] in progress (this will fail) --- awkward-cpp/awkward/cpp/array/array_impl.cpp | 16 ++-- awkward-cpp/awkward/cpp/array/jagged.h | 79 ++++++++++++++++---- tests/test_cpp.py | 75 +++++++++++++++++++ tests/test_numba.py | 10 +-- 4 files changed, 151 insertions(+), 29 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index 5ab2847a..af1a9925 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -2,15 +2,15 @@ PYBIND11_MODULE(array_impl, m) { py::class_(m, "JaggedArray") - .def(py::init()) - .def_property("starts", &JaggedArray::get_starts, &JaggedArray::set_starts) - .def_property("stops", &JaggedArray::get_stops, &JaggedArray::set_stops) + .def(py::init()) + .def_property("starts", &JaggedArray::get_starts, &JaggedArray::python_set_starts) + .def_property("stops", &JaggedArray::get_stops, &JaggedArray::python_set_stops) .def_property("content", &JaggedArray::python_get_content, &JaggedArray::python_set_content) - .def_static("offsets2parents", &JaggedArray::offsets2parents) - .def_static("counts2offsets", &JaggedArray::counts2offsets) - .def_static("startsstops2parents", &JaggedArray::startsstops2parents) - .def_static("parents2startsstops", &JaggedArray::parents2startsstops) - .def_static("uniques2offsetsparents", &JaggedArray::uniques2offsetsparents) + .def_static("offsets2parents", &JaggedArray::python_offsets2parents) + .def_static("counts2offsets", &JaggedArray::python_counts2offsets) + .def_static("startsstops2parents", &JaggedArray::python_startsstops2parents) + .def_static("parents2startsstops", &JaggedArray::python_parents2startsstops) + .def_static("uniques2offsetsparents", &JaggedArray::python_uniques2offsetsparents) .def_static("fromiter", &JaggedArray::fromiter) .def_static("fromoffsets", &JaggedArray::python_fromoffsets) .def_static("fromcounts", &JaggedArray::python_fromcounts) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 60f3d921..3217e4ca 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -75,6 +75,11 @@ class JaggedArray : public AwkwardArray { starts = starts_; } + void python_set_starts(py::object input) { + py::array starts_ = input.cast(); + set_starts(starts_); + } + py::array_t get_stops() { return stops; } void set_stops(py::array stops_) { @@ -93,6 +98,10 @@ class JaggedArray : public AwkwardArray { } stops = stops_; } + void python_set_stops(py::object input) { + py::array stops_ = input.cast(); + set_stops(stops_); + } bool check_validity() { py::buffer_info starts_info = starts.request(); @@ -131,9 +140,9 @@ class JaggedArray : public AwkwardArray { return true; } - JaggedArray(py::array starts_, py::array stops_, py::object content_) { - set_starts(starts_); - set_stops(stops_); + JaggedArray(py::object starts_, py::object stops_, py::object content_) { + python_set_starts(starts_); + python_set_stops(stops_); python_set_content(content_); check_validity(); } @@ -162,7 +171,8 @@ class JaggedArray : public AwkwardArray { ); } - static JaggedArray* python_fromoffsets(py::array offsets, py::object content_) { + static JaggedArray* python_fromoffsets(py::object input, py::object content_) { + py::array offsets = input.cast(); try { return fromoffsets(offsets, content_.cast()); } @@ -179,7 +189,8 @@ class JaggedArray : public AwkwardArray { return fromoffsets(counts2offsets(counts), content_); } - static JaggedArray* python_fromcounts(py::array counts, py::object content_) { + static JaggedArray* python_fromcounts(py::object input, py::object content_) { + py::array counts = input.cast(); try { return fromcounts(counts, content_.cast()); } @@ -241,7 +252,8 @@ class JaggedArray : public AwkwardArray { return new JaggedArray(startsstops[0], startsstops[1], content_); } - static JaggedArray* python_fromparents(py::array parents, py::object content_, ssize_t length = -1) { + static JaggedArray* python_fromparents(py::object input, py::object content_, ssize_t length = -1) { + py::array parents = input.cast(); try { return fromparents(parents, content_.cast(), length); } @@ -262,7 +274,8 @@ class JaggedArray : public AwkwardArray { return fromoffsets(offsetsparents[0], content_); } - static JaggedArray* python_fromuniques(py::array uniques, py::object content_) { + static JaggedArray* python_fromuniques(py::object input, py::object content_) { + py::array uniques = input.cast(); try { return fromuniques(uniques, content_.cast()); } @@ -324,6 +337,11 @@ class JaggedArray : public AwkwardArray { return parents; } + static py::array_t python_offsets2parents(py::object offsetsIter) { + py::array offsets = offsetsIter.cast(); + return offsets2parents(offsets); + } + static py::array_t counts2offsets(py::array counts) { makeIntNative(counts); counts = counts.cast>(); @@ -342,17 +360,21 @@ class JaggedArray : public AwkwardArray { } return offsets; } + static py::array_t python_counts2offsets(py::object countsIter) { + py::array counts = countsIter.cast(); + return counts2offsets(counts); + } - static py::array_t startsstops2parents(py::array starts, py::array stops) { - makeIntNative(starts); - makeIntNative(stops); - starts = starts.cast>(); - stops = stops.cast>(); - py::buffer_info starts_info = starts.request(); + static py::array_t startsstops2parents(py::array starts_, py::array stops_) { + makeIntNative(starts_); + makeIntNative(stops_); + starts_ = starts_.cast>(); + stops_ = stops_.cast>(); + py::buffer_info starts_info = starts_.request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; int N_starts = starts_info.strides[0] / starts_info.itemsize; - py::buffer_info stops_info = stops.request(); + py::buffer_info stops_info = stops_.request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; int N_stops = stops_info.strides[0] / stops_info.itemsize; @@ -384,6 +406,12 @@ class JaggedArray : public AwkwardArray { return parents; } + static py::array_t python_startsstops2parents(py::object startsIter, py::object stopsIter) { + py::array starts_ = startsIter.cast(); + py::array stops_ = stopsIter.cast(); + return startsstops2parents(starts_, stops_); + } + static py::tuple parents2startsstops(py::array parents, std::int64_t length = -1) { makeIntNative(parents); parents = parents.cast>(); @@ -439,6 +467,11 @@ class JaggedArray : public AwkwardArray { return out; } + static py::tuple python_parents2startsstops(py::object parentsIter, std::int64_t length = -1) { + py::array parents = parentsIter.cast(); + return parents2startsstops(parents, length); + } + static py::tuple uniques2offsetsparents(py::array uniques) { makeIntNative(uniques); uniques = uniques.cast>(); @@ -506,6 +539,11 @@ class JaggedArray : public AwkwardArray { return out; } + static py::tuple python_uniques2offsetsparents(py::object uniquesIter) { + py::array uniques = uniquesIter.cast(); + return uniques2offsetsparents(uniques); + } + ssize_t len() { return starts.request().size; } @@ -517,7 +555,9 @@ class JaggedArray : public AwkwardArray { if (length < 0) { throw std::invalid_argument("slice length cannot be less than 0"); } - if (start < 0 || start >= len() || start + (length * step) > len() || start + (length * step) < -1) { + if (start < 0 || start >= len() || (length > 0 && + (start + ((length - 1) * step) > len() || + start + ((length - 1) * step) < -1))) { throw std::out_of_range("getitem must be in the bounds of the array."); } auto newStarts = py::array_t(length); @@ -560,19 +600,26 @@ class JaggedArray : public AwkwardArray { if (stop > length) { stop = length; } + if (start < 0) { + start = 0; + } if (start >= stop) { return getitem(length - 1, 0, step)->unwrap(); } + return getitem(start, (stop + step - start - 1) / step, step)->unwrap(); } else { if (stop < -1) { stop = -1; } + if (start >= length) { + start = length - 1; + } if (start <= stop) { return getitem(0, 0, step)->unwrap(); } + return getitem(start, (stop + step - start + 1) / step, step)->unwrap(); } - return getitem(start, (stop + step - start) / step, step)->unwrap(); } AnyArray* getitem(ssize_t index) { diff --git a/tests/test_cpp.py b/tests/test_cpp.py index 5a21ec1f..61d8f43b 100644 --- a/tests/test_cpp.py +++ b/tests/test_cpp.py @@ -42,6 +42,81 @@ class Test(unittest.TestCase): def runTest(self): pass + def test_cpp_unbox(self): + a = awkward_cpp.JaggedArray.fromiter([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) + a2 = awkward_cpp.JaggedArray.fromcounts([2, 0, 1], a) + def test(x): + return 3.14 + test(a) + test(a2) + + def test_cpp_box(self): + a = awkward_cpp.JaggedArray.fromiter([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) + a2 = awkward_cpp.JaggedArray.fromcounts([2, 0, 1], a) + def test(x): + return x + assert test(a).tolist() == a.tolist() + assert test(a2).tolist() == a2.tolist() + + def test_cpp_init(self): + def test(starts, stops, content): + return awkward_cpp.JaggedArray(starts, stops, content) + starts = numpy.array([0, 3, 3]) + stops = numpy.array([3, 3, 5]) + content = numpy.array([1.1, 2.2, 3.3, 4.4, 5.5]) + z = test(starts, stops, content) + assert z.tolist() == [[1.1, 2.2, 3.3], [], [4.4, 5.5]] + assert z.starts is starts + assert z.stops is stops + assert z.content is content + z = test(starts, stops, content) + assert z.tolist() == [[1.1, 2.2, 3.3], [], [4.4, 5.5]] + assert z.starts is starts + assert z.stops is stops + assert z.content is content + a = awkward_cpp.JaggedArray.fromiter([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) + starts2 = numpy.array([0, 2, 2]) + stops2 = numpy.array([2, 2, 3]) + assert test(starts2, stops2, a).tolist() == [[[1.1, 2.2, 3.3], []], [], [[4.4, 5.5]]] + + def test_cpp_len(self): + a = awkward_cpp.JaggedArray.fromiter([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) + a2 = awkward_cpp.JaggedArray.fromcounts([2, 1], a) + def test1(x): + return len(x) + assert test1(a) == 3 + assert test1(a2) == 2 + + def test_cpp_getitem_integer(self): + a = awkward_cpp.JaggedArray.fromiter([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) + a2 = awkward_cpp.JaggedArray.fromcounts([2, 0, 1], a) + def test1(x, i, j): + return x[i][j] + assert test1(a, 0, 0) == 1.1 + assert test1(a, 0, 1) == 2.2 + assert test1(a, 0, 2) == 3.3 + assert test1(a, 2, 0) == 4.4 + assert test1(a, 2, 1) == 5.5 + def test2(x, i): + return x[i] + assert test2(a, 0).tolist() == [1.1, 2.2, 3.3] + assert test2(a, 1).tolist() == [] + assert test2(a, 2).tolist() == [4.4, 5.5] + assert test2(a2, 0).tolist() == [[1.1, 2.2, 3.3], []] + assert test2(a2, 1).tolist() == [] + assert test2(a2, 2).tolist() == [[4.4, 5.5]] + assert test2(a2, 0).content.tolist() == a.content.tolist() + + def test_cpp_getitem_slice(self): + a = awkward_cpp.JaggedArray.fromiter([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) + a2 = awkward_cpp.JaggedArray.fromcounts([2, 0, 1], a) # [[[1.1, 2.2, 3.3], []], [], [[4.4, 5.5]]] + def test1(x, i, j): + return x[i:j] + assert test1(a, 0, 2).tolist() == [[1.1, 2.2, 3.3], []] + assert test1(a, 1, 3).tolist() == [[], [4.4, 5.5]] + assert test1(a2, 0, 2).tolist() == [[[1.1, 2.2, 3.3], []], []] + assert test1(a2, 1, 3).tolist() == [[], [[4.4, 5.5]]] + def test_cpp_offsets2parents(self): offsets = numpy.array([0, 2, 4, 4, 7], dtype=numpy.int64) parents = awkward_cpp.JaggedArray.offsets2parents(offsets) diff --git a/tests/test_numba.py b/tests/test_numba.py index 2e152611..f4979141 100644 --- a/tests/test_numba.py +++ b/tests/test_numba.py @@ -2,21 +2,21 @@ # Copyright (c) 2019, IRIS-HEP # All rights reserved. -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. -# +# # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# +# # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE From c66d7e5e8ab7f8dd144272a76ac9126b333d0076 Mon Sep 17 00:00:00 2001 From: EscottC Date: Wed, 3 Jul 2019 13:28:35 -0700 Subject: [PATCH 09/26] fixed slices --- awkward-cpp/awkward/cpp/array/array_impl.cpp | 2 +- awkward-cpp/awkward/cpp/array/jagged.h | 39 +++----------------- awkward-cpp/awkward/cpp/array/jagged.py | 18 --------- 3 files changed, 6 insertions(+), 53 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index af1a9925..7bb2651c 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -21,7 +21,7 @@ PYBIND11_MODULE(array_impl, m) { .def("deepcopy", &JaggedArray::deepcopy) .def("tolist", &JaggedArray::tolist) .def("__getitem__", (py::object (JaggedArray::*)(ssize_t)) &JaggedArray::python_getitem) - .def("__getitem__", (py::object (JaggedArray::*)(ssize_t, ssize_t, ssize_t)) &JaggedArray::python_getitem) + .def("__getitem__", (py::object (JaggedArray::*)(py::slice)) &JaggedArray::python_getitem) .def("__str__", &JaggedArray::str) .def("__len__", &JaggedArray::len) .def("__iter__", &JaggedArray::iter) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 3217e4ca..933ed3b0 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -585,41 +585,12 @@ class JaggedArray : public AwkwardArray { return new JaggedArray(newStarts, newStops, content); } - py::object python_getitem(ssize_t start, ssize_t stop, ssize_t step) { - if (step == 0) { - throw std::invalid_argument("slice step cannot be 0"); - } - ssize_t length = len(); - if (start < 0) { - start += length; - } - if (stop < 0) { - stop += length; - } - if (step > 0) { - if (stop > length) { - stop = length; - } - if (start < 0) { - start = 0; - } - if (start >= stop) { - return getitem(length - 1, 0, step)->unwrap(); - } - return getitem(start, (stop + step - start - 1) / step, step)->unwrap(); - } - else { - if (stop < -1) { - stop = -1; - } - if (start >= length) { - start = length - 1; - } - if (start <= stop) { - return getitem(0, 0, step)->unwrap(); - } - return getitem(start, (stop + step - start + 1) / step, step)->unwrap(); + py::object python_getitem(py::slice input) { + size_t start, stop, step, slicelength; + if (!input.compute(len(), &start, &stop, &step, &slicelength)) { + throw py::error_already_set(); } + return getitem((ssize_t)start, (ssize_t)slicelength, (ssize_t)step)->unwrap(); } AnyArray* getitem(ssize_t index) { diff --git a/awkward-cpp/awkward/cpp/array/jagged.py b/awkward-cpp/awkward/cpp/array/jagged.py index 4c279ad2..cdfce337 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.py +++ b/awkward-cpp/awkward/cpp/array/jagged.py @@ -15,21 +15,3 @@ def parents2startsstops(cls, parents, length = None): if length is None: length = -1 return getattr(JaggedArray, "parents2startsstops")(parents, length) - - def __getitem__(self, where): - if isinstance(where, slice): - length = getattr(JaggedArray, "__len__")(self) - start = 0 - stop = length - step = 1 - if where.step is not None: - step = where.step - if step < 0: - start = length - 1 - stop = -1 - length - if where.start is not None: - start = where.start - if where.stop is not None: - stop = where.stop - return getattr(JaggedArray, "__getitem__")(self, start, stop, step) - return getattr(JaggedArray, "__getitem__")(self, where) From a8f97d3f91a66231d3fdd6c96981d22959f43039 Mon Sep 17 00:00:00 2001 From: EscottC Date: Wed, 3 Jul 2019 13:36:53 -0700 Subject: [PATCH 10/26] parents2startsstops to fully C++ side --- awkward-cpp/awkward/cpp/array/array_impl.cpp | 3 ++- awkward-cpp/awkward/cpp/array/jagged.py | 6 +----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index 7bb2651c..3330c234 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -9,7 +9,8 @@ PYBIND11_MODULE(array_impl, m) { .def_static("offsets2parents", &JaggedArray::python_offsets2parents) .def_static("counts2offsets", &JaggedArray::python_counts2offsets) .def_static("startsstops2parents", &JaggedArray::python_startsstops2parents) - .def_static("parents2startsstops", &JaggedArray::python_parents2startsstops) + .def_static("parents2startsstops", &JaggedArray::python_parents2startsstops, + py::arg("parents"), py::arg("length") = -1) .def_static("uniques2offsetsparents", &JaggedArray::python_uniques2offsetsparents) .def_static("fromiter", &JaggedArray::fromiter) .def_static("fromoffsets", &JaggedArray::python_fromoffsets) diff --git a/awkward-cpp/awkward/cpp/array/jagged.py b/awkward-cpp/awkward/cpp/array/jagged.py index cdfce337..bdaac79c 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.py +++ b/awkward-cpp/awkward/cpp/array/jagged.py @@ -10,8 +10,4 @@ from .array_impl import JaggedArray class JaggedArrayCpp(CppMethods, JaggedArray, awkward.array.jagged.JaggedArray): - @classmethod - def parents2startsstops(cls, parents, length = None): - if length is None: - length = -1 - return getattr(JaggedArray, "parents2startsstops")(parents, length) + pass From 57a947371feaf849cb84aa41e93be2ca6e9fec61 Mon Sep 17 00:00:00 2001 From: EscottC Date: Wed, 3 Jul 2019 14:28:17 -0700 Subject: [PATCH 11/26] intarray_getitem and boolarray_getitem --- awkward-cpp/awkward/cpp/array/array_impl.cpp | 1 + awkward-cpp/awkward/cpp/array/jagged.h | 71 ++++++++++++++++++++ tests/test_cpp.py | 52 ++++++++++++++ 3 files changed, 124 insertions(+) diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index 3330c234..c69f0f87 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -23,6 +23,7 @@ PYBIND11_MODULE(array_impl, m) { .def("tolist", &JaggedArray::tolist) .def("__getitem__", (py::object (JaggedArray::*)(ssize_t)) &JaggedArray::python_getitem) .def("__getitem__", (py::object (JaggedArray::*)(py::slice)) &JaggedArray::python_getitem) + .def("__getitem__", (py::object (JaggedArray::*)(py::array)) &JaggedArray::python_getitem) .def("__str__", &JaggedArray::str) .def("__len__", &JaggedArray::len) .def("__iter__", &JaggedArray::iter) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 933ed3b0..e3bc34c8 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -620,6 +620,77 @@ class JaggedArray : public AwkwardArray { return getitem(index)->unwrap(); } + JaggedArray* boolarray_getitem(py::array input) { + ssize_t length = input.request().size; + if (length != len()) { + throw std::invalid_argument("bool array length must be equal to jagged array length"); + } + auto array_ptr = (bool*)input.request().ptr; + + py::list tempStarts; + py::list tempStops; + + py::buffer_info starts_info = starts.request(); + auto starts_ptr = (std::int64_t*)starts_info.ptr; + int N_starts = starts_info.strides[0] / starts_info.itemsize; + + py::buffer_info stops_info = stops.request(); + auto stops_ptr = (std::int64_t*)stops_info.ptr; + int N_stops = stops_info.strides[0] / stops_info.itemsize; + + for (ssize_t i = 0; i < length; i++) { + if (array_ptr[i]) { + tempStarts.append(starts_ptr[i * N_starts]); + tempStops.append(stops_ptr[i * N_stops]); + } + } + py::array_t outStarts = tempStarts.cast>(); + py::array_t outStops = tempStops.cast>(); + return new JaggedArray(outStarts, outStops, content); + } + + JaggedArray* intarray_getitem(py::array input) { + makeIntNative(input); + input = input.cast>(); + py::buffer_info array_info = input.request(); + auto array_ptr = (std::int64_t*)array_info.ptr; + + auto newStarts = py::array_t(array_info.size); + auto newStarts_ptr = (std::int64_t*)newStarts.request().ptr; + + py::buffer_info starts_info = starts.request(); + auto starts_ptr = (std::int64_t*)starts_info.ptr; + int N_starts = starts_info.strides[0] / starts_info.itemsize; + + auto newStops = py::array_t(array_info.size); + auto newStops_ptr = (std::int64_t*)newStops.request().ptr; + + py::buffer_info stops_info = stops.request(); + auto stops_ptr = (std::int64_t*)stops_info.ptr; + int N_stops = stops_info.strides[0] / stops_info.itemsize; + + for (ssize_t i = 0; i < array_info.size; i++) { + std::int64_t here = array_ptr[i]; + if (here < 0 || here >= len()) { + throw std::invalid_argument("int array indices must be within the bounds of the jagged array"); + } + newStarts_ptr[i] = starts_ptr[N_starts * here]; + newStops_ptr[i] = stops_ptr[N_stops * here]; + } + return new JaggedArray(newStarts, newStops, content); + } + + JaggedArray* getitem(py::array input) { + if (input.request().format.find("?") != std::string::npos) { + return boolarray_getitem(input); + } + return intarray_getitem(input); + } + + py::object python_getitem(py::array input) { + return getitem(input)->unwrap(); + } + py::object tolist() { py::list out; for (ssize_t i = 0; i < len(); i++) { diff --git a/tests/test_cpp.py b/tests/test_cpp.py index 61d8f43b..e636f504 100644 --- a/tests/test_cpp.py +++ b/tests/test_cpp.py @@ -117,6 +117,58 @@ def test1(x, i, j): assert test1(a2, 0, 2).tolist() == [[[1.1, 2.2, 3.3], []], []] assert test1(a2, 1, 3).tolist() == [[], [[4.4, 5.5]]] + def test_cpp_getitem_intarray(self): + a = awkward_cpp.JaggedArray.fromiter([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) + starts = numpy.array([0, 3, 4]) + stops = numpy.array([3, 3, 6]) + content = numpy.array([1.1, 2.2, 3.3, 999, 4.4, 5.5]) + a2 = awkward_cpp.JaggedArray(starts, stops, content) + index = numpy.array([2, 2, 0, 1]) + def test1(x, i): + return x[i] + z = test1(a, index) + assert z.tolist() == [[4.4, 5.5], [4.4, 5.5], [1.1, 2.2, 3.3], []] + assert z.content.tolist() == [1.1, 2.2, 3.3, 4.4, 5.5] + z2 = test1(a2, index) + assert z2.tolist() == [[4.4, 5.5], [4.4, 5.5], [1.1, 2.2, 3.3], []] + assert z2.content.tolist() == [1.1, 2.2, 3.3, 999, 4.4, 5.5] + #def test2(x, i): + # return x[i].compact() + #z = test2(a, index) + #assert z.tolist() == [[4.4, 5.5], [4.4, 5.5], [1.1, 2.2, 3.3], []] + #ssert z.content.tolist() == [4.4, 5.5, 4.4, 5.5, 1.1, 2.2, 3.3] + #z2 = test2(a2, index) + #assert z2.tolist() == [[4.4, 5.5], [4.4, 5.5], [1.1, 2.2, 3.3], []] + #assert z2.content.tolist() == [4.4, 5.5, 4.4, 5.5, 1.1, 2.2, 3.3] + #a3 = awkward_cpp.JaggedArray.fromcounts([2, 0, 1], a) + #assert test1(a3, index).tolist() == [[[4.4, 5.5]], [[4.4, 5.5]], [[1.1, 2.2, 3.3], []], []] + + def test_cpp_getitem_boolarray(self): + a = awkward_cpp.JaggedArray.fromiter([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) + starts = numpy.array([0, 3, 4]) + stops = numpy.array([3, 3, 6]) + content = numpy.array([1.1, 2.2, 3.3, 999, 4.4, 5.5]) + a2 = awkward_cpp.JaggedArray(starts, stops, content) + index = numpy.array([False, True, True]) + def test1(x, i): + return x[i] + z = test1(a, index) + assert z.tolist() == [[], [4.4, 5.5]] + assert z.content.tolist() == [1.1, 2.2, 3.3, 4.4, 5.5] + z2 = test1(a2, index) + assert z2.tolist() == [[], [4.4, 5.5]] + assert z2.content.tolist() == [1.1, 2.2, 3.3, 999, 4.4, 5.5] + #def test2(x, i): + # return x[i].compact() + #z = test2(a, index) + #assert z.tolist() == [[], [4.4, 5.5]] + #assert z.content.tolist() == [4.4, 5.5] + #z2 = test2(a2, index) + #assert z2.tolist() == [[], [4.4, 5.5]] + #assert z2.content.tolist() == [4.4, 5.5] + #a3 = awkward_cpp.JaggedArray.fromcounts([2, 0, 1], a) + #assert test1(a3, index).tolist() == [[], [[4.4, 5.5]]] + def test_cpp_offsets2parents(self): offsets = numpy.array([0, 2, 4, 4, 7], dtype=numpy.int64) parents = awkward_cpp.JaggedArray.offsets2parents(offsets) From 011cceb9dd22f0c3cffc50849d5bb04b129d07d4 Mon Sep 17 00:00:00 2001 From: EscottC Date: Tue, 9 Jul 2019 18:50:32 -0700 Subject: [PATCH 12/26] offsets2parents to CPU_methods.h --- awkward-cpp/awkward/cpp/array/CPU_methods.h | 120 +++++++++++++++++++ awkward-cpp/awkward/cpp/array/array_impl.cpp | 4 + awkward-cpp/awkward/cpp/array/jagged.h | 68 ++++++++--- awkward-cpp/awkward/cpp/array/util.h | 16 +-- 4 files changed, 182 insertions(+), 26 deletions(-) create mode 100644 awkward-cpp/awkward/cpp/array/CPU_methods.h diff --git a/awkward-cpp/awkward/cpp/array/CPU_methods.h b/awkward-cpp/awkward/cpp/array/CPU_methods.h new file mode 100644 index 00000000..dad955d3 --- /dev/null +++ b/awkward-cpp/awkward/cpp/array/CPU_methods.h @@ -0,0 +1,120 @@ +#ifdef __cplusplus // specifies that this is C, not C++ +extern "C" { +#endif + +#ifndef CPU_METHODS_H // include guard +#define CPU_METHODS_H + +struct C_array_8 { // single-dimensional? + char *ptr = NULL; + ssize_t itemsize = 0; + ssize_t size = 0; + char byteorder = '='; // '=', '<', or '>' + ssize_t strides = 0; +}; + +struct C_array_16 { + short int *ptr = NULL; + ssize_t itemsize = 0; + ssize_t size = 0; + char byteorder = '='; + ssize_t strides = 0; +}; + +struct C_array_32 { + long int *ptr = NULL; + ssize_t itemsize = 0; + ssize_t size = 0; + char byteorder = '='; + ssize_t strides = 0; +}; + +struct C_array_64 { + long long *ptr = NULL; + ssize_t itemsize = 0; + ssize_t size = 0; + char byteorder = '='; + ssize_t strides = 0; +}; + +int byteswap_16bit(short int *val) { + return (*val = *val << 8 | *val >> 8) ? 1 : 0; +} + +int byteswap_32bit(long int *val) { + *val = ((*val << 8) & 0xFF00FF00) | ((*val >> 8) & 0xFF00FF); + return (*val = (*val << 16) | (*val >> 16)) ? 1 : 0; +} + +int byteswap_64bit(long long *val) { + *val = ((*val << 8) & 0xFF00FF00FF00FF00ULL) | ((*val >> 8) & 0x00FF00FF00FF00FFULL); + *val = ((*val << 16) & 0xFFFF0000FFFF0000ULL) | ((*val >> 16) & 0x0000FFFF0000FFFFULL); + return (*val = (*val << 32) | (*val >> 32)) ? 1 : 0; +} + +int isNative(char input) { // returns true if native, false if non-native + if (input == '=') + return 1; + union { + unsigned long int i; + char c[4]; + } bint = { 0x01020304 }; + return ((bint.c[0] == 1 && input != '<') + || (bint.c[0] != 1 && input != '>')); +} + +int makeNative_16bit(struct C_array_16 *input) { + if (input->itemsize != 2) + return 0; + int N = input->strides / input->itemsize; + if (!isNative(input->byteorder)) + for (ssize_t i = 0; i < input->size; i++) + if (!byteswap_16bit(&(input->ptr[i * N]))) + return 0; + return 1; +} + +int makeNative_32bit(struct C_array_32 *input) { + if (input->itemsize != 4) + return 0; + int N = input->strides / input->itemsize; + if (!isNative(input->byteorder)) + for (ssize_t i = 0; i < input->size; i++) + if (!byteswap_32bit(&(input->ptr[i * N]))) + return 0; + return 1; +} + +int makeNative_64bit(struct C_array_64 *input) { + if (input->itemsize != 8) + return 0; + int N = input->strides / input->itemsize; + if (!isNative(input->byteorder)) + for (ssize_t i = 0; i < input->size; i++) + if (!byteswap_64bit(&(input->ptr[i * N]))) + return 0; + return 1; +} + +int offsets2parents_int64(struct C_array_64 *offsets, struct C_array_64 *parents) { + makeNative_64bit(offsets); + if (offsets->itemsize != 8 || parents->itemsize != 8) + return 0; + int N_off = offsets->strides / offsets->itemsize; + int N_par = parents->strides / parents->itemsize; + ssize_t j = 0; + ssize_t k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < offsets->ptr[i * N_off]) + parents->ptr[N_par * j++] = (long long)k; + k++; + } + return 1; +} + + +#endif // end include guard + +#ifdef __cplusplus // end C compiler instruction +} +#endif diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index c69f0f87..ff3eb209 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -1,5 +1,7 @@ #include "jagged.h" +namespace py = pybind11; + PYBIND11_MODULE(array_impl, m) { py::class_(m, "JaggedArray") .def(py::init()) @@ -24,10 +26,12 @@ PYBIND11_MODULE(array_impl, m) { .def("__getitem__", (py::object (JaggedArray::*)(ssize_t)) &JaggedArray::python_getitem) .def("__getitem__", (py::object (JaggedArray::*)(py::slice)) &JaggedArray::python_getitem) .def("__getitem__", (py::object (JaggedArray::*)(py::array)) &JaggedArray::python_getitem) + .def("__getitem__", (py::object (JaggedArray::*)(py::tuple)) &JaggedArray::python_getitem) .def("__str__", &JaggedArray::str) .def("__len__", &JaggedArray::len) .def("__iter__", &JaggedArray::iter) .def("__repr__", &JaggedArray::repr); + py::class_(m, "JaggedArrayIterator") .def(py::init()) .def("__iter__", &JaggedArray::JaggedArrayIterator::iter) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index e3bc34c8..582d3d7b 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -18,6 +18,7 @@ #include "util.h" #include "any.h" #include "numpytypes.h" +#include "CPU_methods.h" namespace py = pybind11; @@ -308,9 +309,20 @@ class JaggedArray : public AwkwardArray { return (JaggedArray*)deepcopy(); } + // THIS IS NOW AN EXPERIMENTAL ENVIRONMENT + static C_array_64 numpy_to_c64(py::array input) { + input = input.cast>(); + struct C_array_64 temp = { + (std::int64_t*)input.request().ptr, + 8, + input.request().size, + input.request().format.at(0), + input.request().strides[0] + }; + return temp; + } + static py::array_t offsets2parents(py::array offsets) { - makeIntNative(offsets); - offsets = offsets.cast>(); py::buffer_info offsets_info = offsets.request(); if (offsets_info.size <= 0) { throw std::invalid_argument("offsets must have at least one element"); @@ -318,20 +330,11 @@ class JaggedArray : public AwkwardArray { auto offsets_ptr = (std::int64_t*)offsets_info.ptr; int N = offsets_info.strides[0] / offsets_info.itemsize; - ssize_t parents_length = (ssize_t)offsets_ptr[offsets_info.size - 1]; + ssize_t parents_length = (ssize_t)offsets_ptr[(offsets_info.size - 1) * N]; auto parents = py::array_t(parents_length); - py::buffer_info parents_info = parents.request(); - - auto parents_ptr = (std::int64_t*)parents_info.ptr; - ssize_t j = 0; - ssize_t k = -1; - for (ssize_t i = 0; i < offsets_info.size; i++) { - while (j < (ssize_t)offsets_ptr[i * N]) { - parents_ptr[j] = (std::int64_t)k; - j += 1; - } - k += 1; + if (!offsets2parents_int64(&numpy_to_c64(offsets), &numpy_to_c64(parents))) { + throw std::exception("Error in: CPU_methods.h::offsets2parents_int64()"); } return parents; @@ -691,6 +694,43 @@ class JaggedArray : public AwkwardArray { return getitem(input)->unwrap(); } + /*AnyArray* getitem(py::tuple input) { + if (py::len(input) == 0) { + throw std::invalid_argument("getitem requires at least one argument"); + } + if (py::len(input) == 1) { + try { + ssize_t temp = input[0].cast(); + return getitem(temp); + } + catch (py::cast_error e) { } + try { + py::slice temp = input[0].cast(); + return getitem(temp); + } + catch (py::cast_error e) { } + try { + JaggedArray* temp = input[0].cast(); + throw std::invalid_argument("JaggedArray* argument support not yet implemented"); // TODO + } + catch (py::cast_error e) { } + try { + py::array temp = input[0].cast(); + return getitem(temp); + } + catch (py::cast_error e) { + throw std::invalid_argument("argument type not supported for __getitem__"); + } + } + if (py::len(input) == 2) { + + } + }*/ + + py::object python_getitem(py::tuple input) { + return getitem(input)->unwrap(); + } + py::object tolist() { py::list out; for (ssize_t i = 0; i < len(); i++) { diff --git a/awkward-cpp/awkward/cpp/array/util.h b/awkward-cpp/awkward/cpp/array/util.h index 0c9b4c94..55a3a096 100644 --- a/awkward-cpp/awkward/cpp/array/util.h +++ b/awkward-cpp/awkward/cpp/array/util.h @@ -7,6 +7,7 @@ #include #include #include +#include "CPU_methods.h" namespace py = pybind11; @@ -48,31 +49,22 @@ std::int64_t byteswap(std::int64_t val) { return (val << 32) | ((val >> 32) & 0xFFFFFFFFULL); } -bool isNative(py::array input) { - char ch = input.request().format.at(0); - union { - uint32_t i; - char c[4]; - } bint = { 0x01020304 }; - return ((bint.c[0] == 1 && ch != '<') || (bint.c[0] != 1 && ch != '>')); -} - bool isNativeInt(py::array input) { std::string intList = "qQlLhHbB"; if (intList.find(input.request().format.at(0)) == std::string::npos) { throw std::invalid_argument("argument must be of type int"); } - return isNative(input); + return isNative(input.request().format.at(0)); } template void makeNative(py::array_t input) { - if (isNative(input)) { + if (isNative(input.request().format.at(0))) { return; } py::buffer_info array_info = input.request(); auto array_ptr = (T*)array_info.ptr; - int N = array_info.shape[0] / array_info.itemsize; + int N = array_info.strides[0] / array_info.itemsize; for (ssize_t i = 0; i < array_info.size; i++) { array_ptr[i * N] = byteswap(array_ptr[i * N]); From df150df1b55aabefb39062e05e934e3c82577277 Mon Sep 17 00:00:00 2001 From: EscottC Date: Wed, 10 Jul 2019 17:54:12 -0700 Subject: [PATCH 13/26] this is really bad and I'm going to revert this commit --- awkward-cpp/awkward/cpp/array/CPU_methods.h | 129 +++++++++++++++---- awkward-cpp/awkward/cpp/array/CPU_pybind11.h | 119 +++++++++++++++++ awkward-cpp/awkward/cpp/array/array_impl.cpp | 4 +- awkward-cpp/awkward/cpp/array/jagged.h | 111 +++++++--------- awkward-cpp/awkward/cpp/array/numpytypes.h | 3 +- 5 files changed, 279 insertions(+), 87 deletions(-) create mode 100644 awkward-cpp/awkward/cpp/array/CPU_pybind11.h diff --git a/awkward-cpp/awkward/cpp/array/CPU_methods.h b/awkward-cpp/awkward/cpp/array/CPU_methods.h index dad955d3..a0299979 100644 --- a/awkward-cpp/awkward/cpp/array/CPU_methods.h +++ b/awkward-cpp/awkward/cpp/array/CPU_methods.h @@ -5,48 +5,51 @@ extern "C" { #ifndef CPU_METHODS_H // include guard #define CPU_METHODS_H +#include +#include + struct C_array_8 { // single-dimensional? - char *ptr = NULL; + int8_t *ptr = NULL; ssize_t itemsize = 0; ssize_t size = 0; - char byteorder = '='; // '=', '<', or '>' + const char *format = 0; // '=', '<', or '>' ssize_t strides = 0; }; struct C_array_16 { - short int *ptr = NULL; + int16_t *ptr = NULL; ssize_t itemsize = 0; ssize_t size = 0; - char byteorder = '='; + const char *format = 0; ssize_t strides = 0; }; struct C_array_32 { - long int *ptr = NULL; + int32_t *ptr = NULL; ssize_t itemsize = 0; ssize_t size = 0; - char byteorder = '='; + const char *format = 0; ssize_t strides = 0; }; struct C_array_64 { - long long *ptr = NULL; + int64_t *ptr = NULL; ssize_t itemsize = 0; ssize_t size = 0; - char byteorder = '='; + const char *format = 0; ssize_t strides = 0; }; -int byteswap_16bit(short int *val) { +int byteswap_16bit(int16_t *val) { return (*val = *val << 8 | *val >> 8) ? 1 : 0; } -int byteswap_32bit(long int *val) { +int byteswap_32bit(int32_t *val) { *val = ((*val << 8) & 0xFF00FF00) | ((*val >> 8) & 0xFF00FF); return (*val = (*val << 16) | (*val >> 16)) ? 1 : 0; } -int byteswap_64bit(long long *val) { +int byteswap_64bit(int64_t *val) { *val = ((*val << 8) & 0xFF00FF00FF00FF00ULL) | ((*val >> 8) & 0x00FF00FF00FF00FFULL); *val = ((*val << 16) & 0xFFFF0000FFFF0000ULL) | ((*val >> 16) & 0x0000FFFF0000FFFFULL); return (*val = (*val << 32) | (*val >> 32)) ? 1 : 0; @@ -63,11 +66,17 @@ int isNative(char input) { // returns true if native, false if non-native || (bint.c[0] != 1 && input != '>')); } +int isInt(const char *format) { + char *intList = "qQlLhHbB"; + for (ssize_t i = 0; i < 8; i++) + if (intList[i] == format[0] || intList[i] == format[1]) + return 1; + return 0; +} + int makeNative_16bit(struct C_array_16 *input) { - if (input->itemsize != 2) - return 0; int N = input->strides / input->itemsize; - if (!isNative(input->byteorder)) + if (!isNative(input->format[0])) for (ssize_t i = 0; i < input->size; i++) if (!byteswap_16bit(&(input->ptr[i * N]))) return 0; @@ -75,10 +84,8 @@ int makeNative_16bit(struct C_array_16 *input) { } int makeNative_32bit(struct C_array_32 *input) { - if (input->itemsize != 4) - return 0; int N = input->strides / input->itemsize; - if (!isNative(input->byteorder)) + if (!isNative(input->format[0])) for (ssize_t i = 0; i < input->size; i++) if (!byteswap_32bit(&(input->ptr[i * N]))) return 0; @@ -86,19 +93,95 @@ int makeNative_32bit(struct C_array_32 *input) { } int makeNative_64bit(struct C_array_64 *input) { - if (input->itemsize != 8) - return 0; int N = input->strides / input->itemsize; - if (!isNative(input->byteorder)) + if (!isNative(input->format[0])) for (ssize_t i = 0; i < input->size; i++) if (!byteswap_64bit(&(input->ptr[i * N]))) return 0; return 1; } +int checkunsigned2signed_8bit(struct C_array_8 *input) { + int N = input->strides / input->itemsize; + for (ssize_t i = 0; i < input->size; i++) + if ((input->ptr[i * N]) >> 7) + return 0; + return 1; +} + +int checkunsigned2signed_16bit(struct C_array_16 *input) { + int N = input->strides / input->itemsize; + for (ssize_t i = 0; i < input->size; i++) + if ((input->ptr[i * N]) >> 7) + return 0; + return 1; +} + +int checkunsigned2signed_32bit(struct C_array_32 *input) { + int N = input->strides / input->itemsize; + for (ssize_t i = 0; i < input->size; i++) + if ((input->ptr[i * N]) >> 7) + return 0; + return 1; +} + +int checkunsigned2signed_64bit(struct C_array_64 *input) { + int N = input->strides / input->itemsize; + for (ssize_t i = 0; i < input->size; i++) + if ((input->ptr[i * N]) >> 7) + return 0; + return 1; +} + +int checkPos_8bit(struct C_array_8 *input) { + int N = input->strides / input->itemsize; + for (ssize_t i = 0; i < input->size; i++) + if (input->ptr[i * N] < 0) + return 0; + return 1; +} + +int checkPos_16bit(struct C_array_16 *input) { + int N = input->strides / input->itemsize; + for (ssize_t i = 0; i < input->size; i++) + if (input->ptr[i * N] < 0) + return 0; + return 1; +} + +int checkPos_32bit(struct C_array_32 *input) { + int N = input->strides / input->itemsize; + for (ssize_t i = 0; i < input->size; i++) + if (input->ptr[i * N] < 0) + return 0; + return 1; +} + +int checkPos_64bit(struct C_array_64 *input) { + int N = input->strides / input->itemsize; + for (ssize_t i = 0; i < input->size; i++) + if (input->ptr[i * N] < 0) + return 0; + return 1; +} + int offsets2parents_int64(struct C_array_64 *offsets, struct C_array_64 *parents) { - makeNative_64bit(offsets); - if (offsets->itemsize != 8 || parents->itemsize != 8) + if (!makeNative_64bit(offsets)) + return 0; + int N_off = offsets->strides / offsets->itemsize; + int N_par = parents->strides / parents->itemsize; + ssize_t j = 0; + ssize_t k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < offsets->ptr[i * N_off]) + parents->ptr[N_par * j++] = (int64_t)k; + k++; + } + return 1; +} + +int offsets2parents_int32(struct C_array_32 *offsets, struct C_array_32 *parents) { + if (!makeNative_32bit(offsets)) return 0; int N_off = offsets->strides / offsets->itemsize; int N_par = parents->strides / parents->itemsize; @@ -106,7 +189,7 @@ int offsets2parents_int64(struct C_array_64 *offsets, struct C_array_64 *parents ssize_t k = -1; for (ssize_t i = 0; i < offsets->size; i++) { while (j < offsets->ptr[i * N_off]) - parents->ptr[N_par * j++] = (long long)k; + parents->ptr[N_par * j++] = (int32_t)k; k++; } return 1; diff --git a/awkward-cpp/awkward/cpp/array/CPU_pybind11.h b/awkward-cpp/awkward/cpp/array/CPU_pybind11.h new file mode 100644 index 00000000..c96458a9 --- /dev/null +++ b/awkward-cpp/awkward/cpp/array/CPU_pybind11.h @@ -0,0 +1,119 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include "CPU_methods.h" +#include "numpytypes.h" + +C_array_64 numpy2c(NumpyArray_t *input) { + struct C_array_64 temp = { + (std::int64_t*)input->request().ptr, + 8, + input->request().size, + input->request().format.c_str(), + input->request().strides[0] + }; + return temp; +} + +C_array_32 numpy2c(NumpyArray_t *input) { + struct C_array_32 temp = { + (std::int32_t*)input->request().ptr, + 4, + input->request().size, + input->request().format.c_str(), + input->request().strides[0] + }; + return temp; +} + +C_array_16 numpy2c(NumpyArray_t *input) { + struct C_array_16 temp = { + (std::int16_t*)input->request().ptr, + 2, + input->request().size, + input->request().format.c_str(), + input->request().strides[0] + }; + return temp; +} + +C_array_8 numpy2c(NumpyArray_t *input) { + struct C_array_8 temp = { + (std::int8_t*)input->request().ptr, + 1, + input->request().size, + input->request().format.c_str(), + input->request().strides[0] + }; + return temp; +} + +int makeNative_CPU(struct C_array_8 *input) { + return 1; +} + +int makeNative_CPU(struct C_array_16 *input) { + return makeNative_16bit(input); +} + +int makeNative_CPU(struct C_array_32 *input) { + return makeNative_32bit(input); +} + +int makeNative_CPU(struct C_array_64 *input) { + return makeNative_64bit(input); +} + +int makeIntNative_CPU(NumpyArray* input) { + if (!isInt(input->request().format.c_str())) + throw std::invalid_argument("argument must be of type int"); + return makeNative_CPU(numpy2c(getNumpyArray_t(input->unwrap()))); +} + +int checkunsigned2signed_CPU(struct C_array_8 *input) { + return checkunsigned2signed_8bit(input); +} + +int checkunsigned2signed_CPU(struct C_array_16 *input) { + return checkunsigned2signed_16bit(input); +} + +int checkunsigned2signed_CPU(struct C_array_32 *input) { + return checkunsigned2signed_32bit(input); +} + +int checkunsigned2signed_CPU(struct C_array_64 *input) { + return checkunsigned2signed_64bit(input); +} + +int checkPos_CPU(struct C_array_8 *input) { + return checkPos_8bit(input); +} + +int checkPos_CPU(struct C_array_16 *input) { + return checkPos_16bit(input); +} + +int checkPos_CPU(struct C_array_32 *input) { + return checkPos_32bit(input); +} + +int checkPos_CPU(struct C_array_64 *input) { + return checkPos_64bit(input); +} + +int checkPos_CPU(NumpyArray *input) { + return checkPos_CPU(numpy2c(getNumpyArray_t(input->unwrap()))); +} + +int offsets2parents_CPU(C_array_64 *offsets, C_array_64 *parents) { + return offsets2parents_int64(offsets, parents); +} + +int offsets2parents_CPU(C_array_32 *offsets, C_array_32 *parents) { + return offsets2parents_int32(offsets, parents); +} diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index ff3eb209..130e9473 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -5,8 +5,8 @@ namespace py = pybind11; PYBIND11_MODULE(array_impl, m) { py::class_(m, "JaggedArray") .def(py::init()) - .def_property("starts", &JaggedArray::get_starts, &JaggedArray::python_set_starts) - .def_property("stops", &JaggedArray::get_stops, &JaggedArray::python_set_stops) + .def_property("starts", &JaggedArray::python_get_starts, &JaggedArray::python_set_starts) + .def_property("stops", &JaggedArray::python_get_stops, &JaggedArray::python_set_stops) .def_property("content", &JaggedArray::python_get_content, &JaggedArray::python_set_content) .def_static("offsets2parents", &JaggedArray::python_offsets2parents) .def_static("counts2offsets", &JaggedArray::python_counts2offsets) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 582d3d7b..23bf736a 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -19,14 +19,15 @@ #include "any.h" #include "numpytypes.h" #include "CPU_methods.h" +#include "CPU_pybind11.h" namespace py = pybind11; class JaggedArray : public AwkwardArray { public: - py::array_t starts, - stops; - AnyArray* content; + NumpyArray* starts, + stops; + AnyArray* content; py::object unwrap() { return py::cast(this); @@ -57,56 +58,57 @@ class JaggedArray : public AwkwardArray { } } - py::array_t get_starts() { return starts; } + NumpyArray* get_starts() { return starts; } - void set_starts(py::array starts_) { - makeIntNative(starts_); - starts_ = starts_.cast>(); + py::object python_get_starts() { + return starts->unwrap(); + } + + void set_starts(NumpyArray* starts_) { + makeIntNative_CPU(starts_); py::buffer_info starts_info = starts_.request(); if (starts_info.ndim < 1) { throw std::domain_error("starts must have at least 1 dimension"); } - int N = starts_info.strides[0] / starts_info.itemsize; - auto starts_ptr = (std::int64_t*)starts_info.ptr; - for (ssize_t i = 0; i < starts_info.size; i++) { - if (starts_ptr[i * N] < 0) { - throw std::invalid_argument("starts must have all non-negative values: see index [" + std::to_string(i * N) + "]"); - } + if (!checkPos_CPU(starts_)) { + throw std::invalid_argument("starts must have all non-negative values"); } starts = starts_; } void python_set_starts(py::object input) { - py::array starts_ = input.cast(); + NumpyArray* starts_ = getNumpyArray_t(input); set_starts(starts_); } - py::array_t get_stops() { return stops; } + NumpyArray* get_stops() { return stops; } - void set_stops(py::array stops_) { - makeIntNative(stops_); - stops_ = stops_.cast>(); + py::object python_get_stops() { + return stops->unwrap(); + } + + void set_stops(NumpyArray* stops_) { + makeIntNative_CPU(stops_); py::buffer_info stops_info = stops_.request(); if (stops_info.ndim < 1) { throw std::domain_error("stops must have at least 1 dimension"); } - int N = stops_info.strides[0] / stops_info.itemsize; - auto stops_ptr = (std::int64_t*)stops_info.ptr; - for (ssize_t i = 0; i < stops_info.size; i++) { - if (stops_ptr[i * N] < 0) { - throw std::invalid_argument("stops must have all non-negative values: see index [" + std::to_string(i * N) + "]"); - } + if (!checkPos_CPU(stops_)) { + throw std::invalid_argument("stops must have all non-negative values"); } stops = stops_; } + void python_set_stops(py::object input) { - py::array stops_ = input.cast(); + NumpyArray* stops_ = getNumpyArray_t(input); set_stops(stops_); } + // ---------------- everything above this line is updated --------------------- + bool check_validity() { - py::buffer_info starts_info = starts.request(); - py::buffer_info stops_info = stops.request(); + py::buffer_info starts_info = starts->request(); + py::buffer_info stops_info = stops->request(); if (starts_info.size > stops_info.size) { throw std::invalid_argument("starts must have the same (or shorter) length than stops"); } @@ -148,7 +150,7 @@ class JaggedArray : public AwkwardArray { check_validity(); } - JaggedArray(py::array starts_, py::array stops_, AnyArray* content_) { + JaggedArray(NumpyArray* starts_, NumpyArray* stops_, AnyArray* content_) { set_starts(starts_); set_stops(stops_); set_content(content_); @@ -309,34 +311,21 @@ class JaggedArray : public AwkwardArray { return (JaggedArray*)deepcopy(); } - // THIS IS NOW AN EXPERIMENTAL ENVIRONMENT - static C_array_64 numpy_to_c64(py::array input) { - input = input.cast>(); - struct C_array_64 temp = { - (std::int64_t*)input.request().ptr, - 8, - input.request().size, - input.request().format.at(0), - input.request().strides[0] - }; - return temp; - } - - static py::array_t offsets2parents(py::array offsets) { + template // should only be available in int32 and int64 + static NumpyArray_t offsets2parents(NumpyArray_t offsets) { py::buffer_info offsets_info = offsets.request(); if (offsets_info.size <= 0) { throw std::invalid_argument("offsets must have at least one element"); } - auto offsets_ptr = (std::int64_t*)offsets_info.ptr; + auto offsets_ptr = (T*)offsets_info.ptr; int N = offsets_info.strides[0] / offsets_info.itemsize; ssize_t parents_length = (ssize_t)offsets_ptr[(offsets_info.size - 1) * N]; - auto parents = py::array_t(parents_length); + auto parents = py::array_t(parents_length); - if (!offsets2parents_int64(&numpy_to_c64(offsets), &numpy_to_c64(parents))) { - throw std::exception("Error in: CPU_methods.h::offsets2parents_int64()"); + if (!offsets2parents_CPU(&numpy_to_c(offsets), &numpy_to_c(parents))) { + throw std::exception("Error in: CPU_methods.h::offsets2parents_CPU()"); } - return parents; } @@ -433,11 +422,11 @@ class JaggedArray : public AwkwardArray { } auto starts = py::array_t((ssize_t)length); - py::buffer_info starts_info = starts.request(); + py::buffer_info starts_info = starts->request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; auto stops = py::array_t((ssize_t)length); - py::buffer_info stops_info = stops.request(); + py::buffer_info stops_info = stops->request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; for (ssize_t i = 0; i < (ssize_t)length; i++) { @@ -548,7 +537,7 @@ class JaggedArray : public AwkwardArray { } ssize_t len() { - return starts.request().size; + return starts->request().size; } AnyArray* getitem(ssize_t start, ssize_t length, ssize_t step = 1) { @@ -567,7 +556,7 @@ class JaggedArray : public AwkwardArray { py::buffer_info newStarts_info = newStarts.request(); auto newStarts_ptr = (std::int64_t*)newStarts_info.ptr; - py::buffer_info starts_info = starts.request(); + py::buffer_info starts_info = starts->request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; int N_starts = starts_info.strides[0] / starts_info.itemsize; @@ -575,7 +564,7 @@ class JaggedArray : public AwkwardArray { py::buffer_info newStops_info = newStops.request(); auto newStops_ptr = (std::int64_t*)newStops_info.ptr; - py::buffer_info stops_info = stops.request(); + py::buffer_info stops_info = stops->request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; int N_stops = stops_info.strides[0] / stops_info.itemsize; @@ -597,8 +586,8 @@ class JaggedArray : public AwkwardArray { } AnyArray* getitem(ssize_t index) { - py::buffer_info starts_info = starts.request(); - py::buffer_info stops_info = stops.request(); + py::buffer_info starts_info = starts->request(); + py::buffer_info stops_info = stops->request(); if (starts_info.size > stops_info.size) { throw std::out_of_range("starts must have the same or shorter length than stops"); } @@ -618,7 +607,7 @@ class JaggedArray : public AwkwardArray { py::object python_getitem(ssize_t index) { if (index < 0) { - index += starts.request().size; + index += starts->request().size; } return getitem(index)->unwrap(); } @@ -633,11 +622,11 @@ class JaggedArray : public AwkwardArray { py::list tempStarts; py::list tempStops; - py::buffer_info starts_info = starts.request(); + py::buffer_info starts_info = starts->request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; int N_starts = starts_info.strides[0] / starts_info.itemsize; - py::buffer_info stops_info = stops.request(); + py::buffer_info stops_info = stops->request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; int N_stops = stops_info.strides[0] / stops_info.itemsize; @@ -661,14 +650,14 @@ class JaggedArray : public AwkwardArray { auto newStarts = py::array_t(array_info.size); auto newStarts_ptr = (std::int64_t*)newStarts.request().ptr; - py::buffer_info starts_info = starts.request(); + py::buffer_info starts_info = starts->request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; int N_starts = starts_info.strides[0] / starts_info.itemsize; auto newStops = py::array_t(array_info.size); auto newStops_ptr = (std::int64_t*)newStops.request().ptr; - py::buffer_info stops_info = stops.request(); + py::buffer_info stops_info = stops->request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; int N_stops = stops_info.strides[0] / stops_info.itemsize; @@ -742,10 +731,10 @@ class JaggedArray : public AwkwardArray { std::string str() { std::string out; - py::buffer_info starts_info = starts.request(); + py::buffer_info starts_info = starts->request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; - py::buffer_info stops_info = stops.request(); + py::buffer_info stops_info = stops->request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; out.reserve(starts_info.size * 20); diff --git a/awkward-cpp/awkward/cpp/array/numpytypes.h b/awkward-cpp/awkward/cpp/array/numpytypes.h index fb7e2cd8..230d6766 100644 --- a/awkward-cpp/awkward/cpp/array/numpytypes.h +++ b/awkward-cpp/awkward/cpp/array/numpytypes.h @@ -82,7 +82,8 @@ class NumpyArray_t : public NumpyArray { py::buffer_info request() { return thisArray.request(); } }; -NumpyArray* getNumpyArray_t(py::array input) { +NumpyArray* getNumpyArray_t(py::object iter) { + py::array input = iter.cast(); std::string format = input.request().format; if (format.find("q") != std::string::npos) return new NumpyArray_t(input.cast>()); From 9a18c4405c42493bd9c4c4afa99c0961af05b1a2 Mon Sep 17 00:00:00 2001 From: EscottC Date: Wed, 10 Jul 2019 17:54:33 -0700 Subject: [PATCH 14/26] Revert "this is really bad and I'm going to revert this commit" This reverts commit df150df1b55aabefb39062e05e934e3c82577277. --- awkward-cpp/awkward/cpp/array/CPU_methods.h | 129 ++++--------------- awkward-cpp/awkward/cpp/array/CPU_pybind11.h | 119 ----------------- awkward-cpp/awkward/cpp/array/array_impl.cpp | 4 +- awkward-cpp/awkward/cpp/array/jagged.h | 111 +++++++++------- awkward-cpp/awkward/cpp/array/numpytypes.h | 3 +- 5 files changed, 87 insertions(+), 279 deletions(-) delete mode 100644 awkward-cpp/awkward/cpp/array/CPU_pybind11.h diff --git a/awkward-cpp/awkward/cpp/array/CPU_methods.h b/awkward-cpp/awkward/cpp/array/CPU_methods.h index a0299979..dad955d3 100644 --- a/awkward-cpp/awkward/cpp/array/CPU_methods.h +++ b/awkward-cpp/awkward/cpp/array/CPU_methods.h @@ -5,51 +5,48 @@ extern "C" { #ifndef CPU_METHODS_H // include guard #define CPU_METHODS_H -#include -#include - struct C_array_8 { // single-dimensional? - int8_t *ptr = NULL; + char *ptr = NULL; ssize_t itemsize = 0; ssize_t size = 0; - const char *format = 0; // '=', '<', or '>' + char byteorder = '='; // '=', '<', or '>' ssize_t strides = 0; }; struct C_array_16 { - int16_t *ptr = NULL; + short int *ptr = NULL; ssize_t itemsize = 0; ssize_t size = 0; - const char *format = 0; + char byteorder = '='; ssize_t strides = 0; }; struct C_array_32 { - int32_t *ptr = NULL; + long int *ptr = NULL; ssize_t itemsize = 0; ssize_t size = 0; - const char *format = 0; + char byteorder = '='; ssize_t strides = 0; }; struct C_array_64 { - int64_t *ptr = NULL; + long long *ptr = NULL; ssize_t itemsize = 0; ssize_t size = 0; - const char *format = 0; + char byteorder = '='; ssize_t strides = 0; }; -int byteswap_16bit(int16_t *val) { +int byteswap_16bit(short int *val) { return (*val = *val << 8 | *val >> 8) ? 1 : 0; } -int byteswap_32bit(int32_t *val) { +int byteswap_32bit(long int *val) { *val = ((*val << 8) & 0xFF00FF00) | ((*val >> 8) & 0xFF00FF); return (*val = (*val << 16) | (*val >> 16)) ? 1 : 0; } -int byteswap_64bit(int64_t *val) { +int byteswap_64bit(long long *val) { *val = ((*val << 8) & 0xFF00FF00FF00FF00ULL) | ((*val >> 8) & 0x00FF00FF00FF00FFULL); *val = ((*val << 16) & 0xFFFF0000FFFF0000ULL) | ((*val >> 16) & 0x0000FFFF0000FFFFULL); return (*val = (*val << 32) | (*val >> 32)) ? 1 : 0; @@ -66,17 +63,11 @@ int isNative(char input) { // returns true if native, false if non-native || (bint.c[0] != 1 && input != '>')); } -int isInt(const char *format) { - char *intList = "qQlLhHbB"; - for (ssize_t i = 0; i < 8; i++) - if (intList[i] == format[0] || intList[i] == format[1]) - return 1; - return 0; -} - int makeNative_16bit(struct C_array_16 *input) { + if (input->itemsize != 2) + return 0; int N = input->strides / input->itemsize; - if (!isNative(input->format[0])) + if (!isNative(input->byteorder)) for (ssize_t i = 0; i < input->size; i++) if (!byteswap_16bit(&(input->ptr[i * N]))) return 0; @@ -84,8 +75,10 @@ int makeNative_16bit(struct C_array_16 *input) { } int makeNative_32bit(struct C_array_32 *input) { + if (input->itemsize != 4) + return 0; int N = input->strides / input->itemsize; - if (!isNative(input->format[0])) + if (!isNative(input->byteorder)) for (ssize_t i = 0; i < input->size; i++) if (!byteswap_32bit(&(input->ptr[i * N]))) return 0; @@ -93,95 +86,19 @@ int makeNative_32bit(struct C_array_32 *input) { } int makeNative_64bit(struct C_array_64 *input) { + if (input->itemsize != 8) + return 0; int N = input->strides / input->itemsize; - if (!isNative(input->format[0])) + if (!isNative(input->byteorder)) for (ssize_t i = 0; i < input->size; i++) if (!byteswap_64bit(&(input->ptr[i * N]))) return 0; return 1; } -int checkunsigned2signed_8bit(struct C_array_8 *input) { - int N = input->strides / input->itemsize; - for (ssize_t i = 0; i < input->size; i++) - if ((input->ptr[i * N]) >> 7) - return 0; - return 1; -} - -int checkunsigned2signed_16bit(struct C_array_16 *input) { - int N = input->strides / input->itemsize; - for (ssize_t i = 0; i < input->size; i++) - if ((input->ptr[i * N]) >> 7) - return 0; - return 1; -} - -int checkunsigned2signed_32bit(struct C_array_32 *input) { - int N = input->strides / input->itemsize; - for (ssize_t i = 0; i < input->size; i++) - if ((input->ptr[i * N]) >> 7) - return 0; - return 1; -} - -int checkunsigned2signed_64bit(struct C_array_64 *input) { - int N = input->strides / input->itemsize; - for (ssize_t i = 0; i < input->size; i++) - if ((input->ptr[i * N]) >> 7) - return 0; - return 1; -} - -int checkPos_8bit(struct C_array_8 *input) { - int N = input->strides / input->itemsize; - for (ssize_t i = 0; i < input->size; i++) - if (input->ptr[i * N] < 0) - return 0; - return 1; -} - -int checkPos_16bit(struct C_array_16 *input) { - int N = input->strides / input->itemsize; - for (ssize_t i = 0; i < input->size; i++) - if (input->ptr[i * N] < 0) - return 0; - return 1; -} - -int checkPos_32bit(struct C_array_32 *input) { - int N = input->strides / input->itemsize; - for (ssize_t i = 0; i < input->size; i++) - if (input->ptr[i * N] < 0) - return 0; - return 1; -} - -int checkPos_64bit(struct C_array_64 *input) { - int N = input->strides / input->itemsize; - for (ssize_t i = 0; i < input->size; i++) - if (input->ptr[i * N] < 0) - return 0; - return 1; -} - int offsets2parents_int64(struct C_array_64 *offsets, struct C_array_64 *parents) { - if (!makeNative_64bit(offsets)) - return 0; - int N_off = offsets->strides / offsets->itemsize; - int N_par = parents->strides / parents->itemsize; - ssize_t j = 0; - ssize_t k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < offsets->ptr[i * N_off]) - parents->ptr[N_par * j++] = (int64_t)k; - k++; - } - return 1; -} - -int offsets2parents_int32(struct C_array_32 *offsets, struct C_array_32 *parents) { - if (!makeNative_32bit(offsets)) + makeNative_64bit(offsets); + if (offsets->itemsize != 8 || parents->itemsize != 8) return 0; int N_off = offsets->strides / offsets->itemsize; int N_par = parents->strides / parents->itemsize; @@ -189,7 +106,7 @@ int offsets2parents_int32(struct C_array_32 *offsets, struct C_array_32 *parents ssize_t k = -1; for (ssize_t i = 0; i < offsets->size; i++) { while (j < offsets->ptr[i * N_off]) - parents->ptr[N_par * j++] = (int32_t)k; + parents->ptr[N_par * j++] = (long long)k; k++; } return 1; diff --git a/awkward-cpp/awkward/cpp/array/CPU_pybind11.h b/awkward-cpp/awkward/cpp/array/CPU_pybind11.h deleted file mode 100644 index c96458a9..00000000 --- a/awkward-cpp/awkward/cpp/array/CPU_pybind11.h +++ /dev/null @@ -1,119 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include -#include -#include "CPU_methods.h" -#include "numpytypes.h" - -C_array_64 numpy2c(NumpyArray_t *input) { - struct C_array_64 temp = { - (std::int64_t*)input->request().ptr, - 8, - input->request().size, - input->request().format.c_str(), - input->request().strides[0] - }; - return temp; -} - -C_array_32 numpy2c(NumpyArray_t *input) { - struct C_array_32 temp = { - (std::int32_t*)input->request().ptr, - 4, - input->request().size, - input->request().format.c_str(), - input->request().strides[0] - }; - return temp; -} - -C_array_16 numpy2c(NumpyArray_t *input) { - struct C_array_16 temp = { - (std::int16_t*)input->request().ptr, - 2, - input->request().size, - input->request().format.c_str(), - input->request().strides[0] - }; - return temp; -} - -C_array_8 numpy2c(NumpyArray_t *input) { - struct C_array_8 temp = { - (std::int8_t*)input->request().ptr, - 1, - input->request().size, - input->request().format.c_str(), - input->request().strides[0] - }; - return temp; -} - -int makeNative_CPU(struct C_array_8 *input) { - return 1; -} - -int makeNative_CPU(struct C_array_16 *input) { - return makeNative_16bit(input); -} - -int makeNative_CPU(struct C_array_32 *input) { - return makeNative_32bit(input); -} - -int makeNative_CPU(struct C_array_64 *input) { - return makeNative_64bit(input); -} - -int makeIntNative_CPU(NumpyArray* input) { - if (!isInt(input->request().format.c_str())) - throw std::invalid_argument("argument must be of type int"); - return makeNative_CPU(numpy2c(getNumpyArray_t(input->unwrap()))); -} - -int checkunsigned2signed_CPU(struct C_array_8 *input) { - return checkunsigned2signed_8bit(input); -} - -int checkunsigned2signed_CPU(struct C_array_16 *input) { - return checkunsigned2signed_16bit(input); -} - -int checkunsigned2signed_CPU(struct C_array_32 *input) { - return checkunsigned2signed_32bit(input); -} - -int checkunsigned2signed_CPU(struct C_array_64 *input) { - return checkunsigned2signed_64bit(input); -} - -int checkPos_CPU(struct C_array_8 *input) { - return checkPos_8bit(input); -} - -int checkPos_CPU(struct C_array_16 *input) { - return checkPos_16bit(input); -} - -int checkPos_CPU(struct C_array_32 *input) { - return checkPos_32bit(input); -} - -int checkPos_CPU(struct C_array_64 *input) { - return checkPos_64bit(input); -} - -int checkPos_CPU(NumpyArray *input) { - return checkPos_CPU(numpy2c(getNumpyArray_t(input->unwrap()))); -} - -int offsets2parents_CPU(C_array_64 *offsets, C_array_64 *parents) { - return offsets2parents_int64(offsets, parents); -} - -int offsets2parents_CPU(C_array_32 *offsets, C_array_32 *parents) { - return offsets2parents_int32(offsets, parents); -} diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index 130e9473..ff3eb209 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -5,8 +5,8 @@ namespace py = pybind11; PYBIND11_MODULE(array_impl, m) { py::class_(m, "JaggedArray") .def(py::init()) - .def_property("starts", &JaggedArray::python_get_starts, &JaggedArray::python_set_starts) - .def_property("stops", &JaggedArray::python_get_stops, &JaggedArray::python_set_stops) + .def_property("starts", &JaggedArray::get_starts, &JaggedArray::python_set_starts) + .def_property("stops", &JaggedArray::get_stops, &JaggedArray::python_set_stops) .def_property("content", &JaggedArray::python_get_content, &JaggedArray::python_set_content) .def_static("offsets2parents", &JaggedArray::python_offsets2parents) .def_static("counts2offsets", &JaggedArray::python_counts2offsets) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 23bf736a..582d3d7b 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -19,15 +19,14 @@ #include "any.h" #include "numpytypes.h" #include "CPU_methods.h" -#include "CPU_pybind11.h" namespace py = pybind11; class JaggedArray : public AwkwardArray { public: - NumpyArray* starts, - stops; - AnyArray* content; + py::array_t starts, + stops; + AnyArray* content; py::object unwrap() { return py::cast(this); @@ -58,57 +57,56 @@ class JaggedArray : public AwkwardArray { } } - NumpyArray* get_starts() { return starts; } + py::array_t get_starts() { return starts; } - py::object python_get_starts() { - return starts->unwrap(); - } - - void set_starts(NumpyArray* starts_) { - makeIntNative_CPU(starts_); + void set_starts(py::array starts_) { + makeIntNative(starts_); + starts_ = starts_.cast>(); py::buffer_info starts_info = starts_.request(); if (starts_info.ndim < 1) { throw std::domain_error("starts must have at least 1 dimension"); } - if (!checkPos_CPU(starts_)) { - throw std::invalid_argument("starts must have all non-negative values"); + int N = starts_info.strides[0] / starts_info.itemsize; + auto starts_ptr = (std::int64_t*)starts_info.ptr; + for (ssize_t i = 0; i < starts_info.size; i++) { + if (starts_ptr[i * N] < 0) { + throw std::invalid_argument("starts must have all non-negative values: see index [" + std::to_string(i * N) + "]"); + } } starts = starts_; } void python_set_starts(py::object input) { - NumpyArray* starts_ = getNumpyArray_t(input); + py::array starts_ = input.cast(); set_starts(starts_); } - NumpyArray* get_stops() { return stops; } + py::array_t get_stops() { return stops; } - py::object python_get_stops() { - return stops->unwrap(); - } - - void set_stops(NumpyArray* stops_) { - makeIntNative_CPU(stops_); + void set_stops(py::array stops_) { + makeIntNative(stops_); + stops_ = stops_.cast>(); py::buffer_info stops_info = stops_.request(); if (stops_info.ndim < 1) { throw std::domain_error("stops must have at least 1 dimension"); } - if (!checkPos_CPU(stops_)) { - throw std::invalid_argument("stops must have all non-negative values"); + int N = stops_info.strides[0] / stops_info.itemsize; + auto stops_ptr = (std::int64_t*)stops_info.ptr; + for (ssize_t i = 0; i < stops_info.size; i++) { + if (stops_ptr[i * N] < 0) { + throw std::invalid_argument("stops must have all non-negative values: see index [" + std::to_string(i * N) + "]"); + } } stops = stops_; } - void python_set_stops(py::object input) { - NumpyArray* stops_ = getNumpyArray_t(input); + py::array stops_ = input.cast(); set_stops(stops_); } - // ---------------- everything above this line is updated --------------------- - bool check_validity() { - py::buffer_info starts_info = starts->request(); - py::buffer_info stops_info = stops->request(); + py::buffer_info starts_info = starts.request(); + py::buffer_info stops_info = stops.request(); if (starts_info.size > stops_info.size) { throw std::invalid_argument("starts must have the same (or shorter) length than stops"); } @@ -150,7 +148,7 @@ class JaggedArray : public AwkwardArray { check_validity(); } - JaggedArray(NumpyArray* starts_, NumpyArray* stops_, AnyArray* content_) { + JaggedArray(py::array starts_, py::array stops_, AnyArray* content_) { set_starts(starts_); set_stops(stops_); set_content(content_); @@ -311,21 +309,34 @@ class JaggedArray : public AwkwardArray { return (JaggedArray*)deepcopy(); } - template // should only be available in int32 and int64 - static NumpyArray_t offsets2parents(NumpyArray_t offsets) { + // THIS IS NOW AN EXPERIMENTAL ENVIRONMENT + static C_array_64 numpy_to_c64(py::array input) { + input = input.cast>(); + struct C_array_64 temp = { + (std::int64_t*)input.request().ptr, + 8, + input.request().size, + input.request().format.at(0), + input.request().strides[0] + }; + return temp; + } + + static py::array_t offsets2parents(py::array offsets) { py::buffer_info offsets_info = offsets.request(); if (offsets_info.size <= 0) { throw std::invalid_argument("offsets must have at least one element"); } - auto offsets_ptr = (T*)offsets_info.ptr; + auto offsets_ptr = (std::int64_t*)offsets_info.ptr; int N = offsets_info.strides[0] / offsets_info.itemsize; ssize_t parents_length = (ssize_t)offsets_ptr[(offsets_info.size - 1) * N]; - auto parents = py::array_t(parents_length); + auto parents = py::array_t(parents_length); - if (!offsets2parents_CPU(&numpy_to_c(offsets), &numpy_to_c(parents))) { - throw std::exception("Error in: CPU_methods.h::offsets2parents_CPU()"); + if (!offsets2parents_int64(&numpy_to_c64(offsets), &numpy_to_c64(parents))) { + throw std::exception("Error in: CPU_methods.h::offsets2parents_int64()"); } + return parents; } @@ -422,11 +433,11 @@ class JaggedArray : public AwkwardArray { } auto starts = py::array_t((ssize_t)length); - py::buffer_info starts_info = starts->request(); + py::buffer_info starts_info = starts.request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; auto stops = py::array_t((ssize_t)length); - py::buffer_info stops_info = stops->request(); + py::buffer_info stops_info = stops.request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; for (ssize_t i = 0; i < (ssize_t)length; i++) { @@ -537,7 +548,7 @@ class JaggedArray : public AwkwardArray { } ssize_t len() { - return starts->request().size; + return starts.request().size; } AnyArray* getitem(ssize_t start, ssize_t length, ssize_t step = 1) { @@ -556,7 +567,7 @@ class JaggedArray : public AwkwardArray { py::buffer_info newStarts_info = newStarts.request(); auto newStarts_ptr = (std::int64_t*)newStarts_info.ptr; - py::buffer_info starts_info = starts->request(); + py::buffer_info starts_info = starts.request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; int N_starts = starts_info.strides[0] / starts_info.itemsize; @@ -564,7 +575,7 @@ class JaggedArray : public AwkwardArray { py::buffer_info newStops_info = newStops.request(); auto newStops_ptr = (std::int64_t*)newStops_info.ptr; - py::buffer_info stops_info = stops->request(); + py::buffer_info stops_info = stops.request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; int N_stops = stops_info.strides[0] / stops_info.itemsize; @@ -586,8 +597,8 @@ class JaggedArray : public AwkwardArray { } AnyArray* getitem(ssize_t index) { - py::buffer_info starts_info = starts->request(); - py::buffer_info stops_info = stops->request(); + py::buffer_info starts_info = starts.request(); + py::buffer_info stops_info = stops.request(); if (starts_info.size > stops_info.size) { throw std::out_of_range("starts must have the same or shorter length than stops"); } @@ -607,7 +618,7 @@ class JaggedArray : public AwkwardArray { py::object python_getitem(ssize_t index) { if (index < 0) { - index += starts->request().size; + index += starts.request().size; } return getitem(index)->unwrap(); } @@ -622,11 +633,11 @@ class JaggedArray : public AwkwardArray { py::list tempStarts; py::list tempStops; - py::buffer_info starts_info = starts->request(); + py::buffer_info starts_info = starts.request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; int N_starts = starts_info.strides[0] / starts_info.itemsize; - py::buffer_info stops_info = stops->request(); + py::buffer_info stops_info = stops.request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; int N_stops = stops_info.strides[0] / stops_info.itemsize; @@ -650,14 +661,14 @@ class JaggedArray : public AwkwardArray { auto newStarts = py::array_t(array_info.size); auto newStarts_ptr = (std::int64_t*)newStarts.request().ptr; - py::buffer_info starts_info = starts->request(); + py::buffer_info starts_info = starts.request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; int N_starts = starts_info.strides[0] / starts_info.itemsize; auto newStops = py::array_t(array_info.size); auto newStops_ptr = (std::int64_t*)newStops.request().ptr; - py::buffer_info stops_info = stops->request(); + py::buffer_info stops_info = stops.request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; int N_stops = stops_info.strides[0] / stops_info.itemsize; @@ -731,10 +742,10 @@ class JaggedArray : public AwkwardArray { std::string str() { std::string out; - py::buffer_info starts_info = starts->request(); + py::buffer_info starts_info = starts.request(); auto starts_ptr = (std::int64_t*)starts_info.ptr; - py::buffer_info stops_info = stops->request(); + py::buffer_info stops_info = stops.request(); auto stops_ptr = (std::int64_t*)stops_info.ptr; out.reserve(starts_info.size * 20); diff --git a/awkward-cpp/awkward/cpp/array/numpytypes.h b/awkward-cpp/awkward/cpp/array/numpytypes.h index 230d6766..fb7e2cd8 100644 --- a/awkward-cpp/awkward/cpp/array/numpytypes.h +++ b/awkward-cpp/awkward/cpp/array/numpytypes.h @@ -82,8 +82,7 @@ class NumpyArray_t : public NumpyArray { py::buffer_info request() { return thisArray.request(); } }; -NumpyArray* getNumpyArray_t(py::object iter) { - py::array input = iter.cast(); +NumpyArray* getNumpyArray_t(py::array input) { std::string format = input.request().format; if (format.find("q") != std::string::npos) return new NumpyArray_t(input.cast>()); From 7c2e8d4d0affd1eafe77e7e68be69b2cf2c205b4 Mon Sep 17 00:00:00 2001 From: EscottC Date: Wed, 10 Jul 2019 17:55:00 -0700 Subject: [PATCH 15/26] Revert "offsets2parents to CPU_methods.h" This reverts commit 011cceb9dd22f0c3cffc50849d5bb04b129d07d4. --- awkward-cpp/awkward/cpp/array/CPU_methods.h | 120 ------------------- awkward-cpp/awkward/cpp/array/array_impl.cpp | 4 - awkward-cpp/awkward/cpp/array/jagged.h | 68 +++-------- awkward-cpp/awkward/cpp/array/util.h | 16 ++- 4 files changed, 26 insertions(+), 182 deletions(-) delete mode 100644 awkward-cpp/awkward/cpp/array/CPU_methods.h diff --git a/awkward-cpp/awkward/cpp/array/CPU_methods.h b/awkward-cpp/awkward/cpp/array/CPU_methods.h deleted file mode 100644 index dad955d3..00000000 --- a/awkward-cpp/awkward/cpp/array/CPU_methods.h +++ /dev/null @@ -1,120 +0,0 @@ -#ifdef __cplusplus // specifies that this is C, not C++ -extern "C" { -#endif - -#ifndef CPU_METHODS_H // include guard -#define CPU_METHODS_H - -struct C_array_8 { // single-dimensional? - char *ptr = NULL; - ssize_t itemsize = 0; - ssize_t size = 0; - char byteorder = '='; // '=', '<', or '>' - ssize_t strides = 0; -}; - -struct C_array_16 { - short int *ptr = NULL; - ssize_t itemsize = 0; - ssize_t size = 0; - char byteorder = '='; - ssize_t strides = 0; -}; - -struct C_array_32 { - long int *ptr = NULL; - ssize_t itemsize = 0; - ssize_t size = 0; - char byteorder = '='; - ssize_t strides = 0; -}; - -struct C_array_64 { - long long *ptr = NULL; - ssize_t itemsize = 0; - ssize_t size = 0; - char byteorder = '='; - ssize_t strides = 0; -}; - -int byteswap_16bit(short int *val) { - return (*val = *val << 8 | *val >> 8) ? 1 : 0; -} - -int byteswap_32bit(long int *val) { - *val = ((*val << 8) & 0xFF00FF00) | ((*val >> 8) & 0xFF00FF); - return (*val = (*val << 16) | (*val >> 16)) ? 1 : 0; -} - -int byteswap_64bit(long long *val) { - *val = ((*val << 8) & 0xFF00FF00FF00FF00ULL) | ((*val >> 8) & 0x00FF00FF00FF00FFULL); - *val = ((*val << 16) & 0xFFFF0000FFFF0000ULL) | ((*val >> 16) & 0x0000FFFF0000FFFFULL); - return (*val = (*val << 32) | (*val >> 32)) ? 1 : 0; -} - -int isNative(char input) { // returns true if native, false if non-native - if (input == '=') - return 1; - union { - unsigned long int i; - char c[4]; - } bint = { 0x01020304 }; - return ((bint.c[0] == 1 && input != '<') - || (bint.c[0] != 1 && input != '>')); -} - -int makeNative_16bit(struct C_array_16 *input) { - if (input->itemsize != 2) - return 0; - int N = input->strides / input->itemsize; - if (!isNative(input->byteorder)) - for (ssize_t i = 0; i < input->size; i++) - if (!byteswap_16bit(&(input->ptr[i * N]))) - return 0; - return 1; -} - -int makeNative_32bit(struct C_array_32 *input) { - if (input->itemsize != 4) - return 0; - int N = input->strides / input->itemsize; - if (!isNative(input->byteorder)) - for (ssize_t i = 0; i < input->size; i++) - if (!byteswap_32bit(&(input->ptr[i * N]))) - return 0; - return 1; -} - -int makeNative_64bit(struct C_array_64 *input) { - if (input->itemsize != 8) - return 0; - int N = input->strides / input->itemsize; - if (!isNative(input->byteorder)) - for (ssize_t i = 0; i < input->size; i++) - if (!byteswap_64bit(&(input->ptr[i * N]))) - return 0; - return 1; -} - -int offsets2parents_int64(struct C_array_64 *offsets, struct C_array_64 *parents) { - makeNative_64bit(offsets); - if (offsets->itemsize != 8 || parents->itemsize != 8) - return 0; - int N_off = offsets->strides / offsets->itemsize; - int N_par = parents->strides / parents->itemsize; - ssize_t j = 0; - ssize_t k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < offsets->ptr[i * N_off]) - parents->ptr[N_par * j++] = (long long)k; - k++; - } - return 1; -} - - -#endif // end include guard - -#ifdef __cplusplus // end C compiler instruction -} -#endif diff --git a/awkward-cpp/awkward/cpp/array/array_impl.cpp b/awkward-cpp/awkward/cpp/array/array_impl.cpp index ff3eb209..c69f0f87 100644 --- a/awkward-cpp/awkward/cpp/array/array_impl.cpp +++ b/awkward-cpp/awkward/cpp/array/array_impl.cpp @@ -1,7 +1,5 @@ #include "jagged.h" -namespace py = pybind11; - PYBIND11_MODULE(array_impl, m) { py::class_(m, "JaggedArray") .def(py::init()) @@ -26,12 +24,10 @@ PYBIND11_MODULE(array_impl, m) { .def("__getitem__", (py::object (JaggedArray::*)(ssize_t)) &JaggedArray::python_getitem) .def("__getitem__", (py::object (JaggedArray::*)(py::slice)) &JaggedArray::python_getitem) .def("__getitem__", (py::object (JaggedArray::*)(py::array)) &JaggedArray::python_getitem) - .def("__getitem__", (py::object (JaggedArray::*)(py::tuple)) &JaggedArray::python_getitem) .def("__str__", &JaggedArray::str) .def("__len__", &JaggedArray::len) .def("__iter__", &JaggedArray::iter) .def("__repr__", &JaggedArray::repr); - py::class_(m, "JaggedArrayIterator") .def(py::init()) .def("__iter__", &JaggedArray::JaggedArrayIterator::iter) diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 582d3d7b..e3bc34c8 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -18,7 +18,6 @@ #include "util.h" #include "any.h" #include "numpytypes.h" -#include "CPU_methods.h" namespace py = pybind11; @@ -309,20 +308,9 @@ class JaggedArray : public AwkwardArray { return (JaggedArray*)deepcopy(); } - // THIS IS NOW AN EXPERIMENTAL ENVIRONMENT - static C_array_64 numpy_to_c64(py::array input) { - input = input.cast>(); - struct C_array_64 temp = { - (std::int64_t*)input.request().ptr, - 8, - input.request().size, - input.request().format.at(0), - input.request().strides[0] - }; - return temp; - } - static py::array_t offsets2parents(py::array offsets) { + makeIntNative(offsets); + offsets = offsets.cast>(); py::buffer_info offsets_info = offsets.request(); if (offsets_info.size <= 0) { throw std::invalid_argument("offsets must have at least one element"); @@ -330,11 +318,20 @@ class JaggedArray : public AwkwardArray { auto offsets_ptr = (std::int64_t*)offsets_info.ptr; int N = offsets_info.strides[0] / offsets_info.itemsize; - ssize_t parents_length = (ssize_t)offsets_ptr[(offsets_info.size - 1) * N]; + ssize_t parents_length = (ssize_t)offsets_ptr[offsets_info.size - 1]; auto parents = py::array_t(parents_length); + py::buffer_info parents_info = parents.request(); + + auto parents_ptr = (std::int64_t*)parents_info.ptr; - if (!offsets2parents_int64(&numpy_to_c64(offsets), &numpy_to_c64(parents))) { - throw std::exception("Error in: CPU_methods.h::offsets2parents_int64()"); + ssize_t j = 0; + ssize_t k = -1; + for (ssize_t i = 0; i < offsets_info.size; i++) { + while (j < (ssize_t)offsets_ptr[i * N]) { + parents_ptr[j] = (std::int64_t)k; + j += 1; + } + k += 1; } return parents; @@ -694,43 +691,6 @@ class JaggedArray : public AwkwardArray { return getitem(input)->unwrap(); } - /*AnyArray* getitem(py::tuple input) { - if (py::len(input) == 0) { - throw std::invalid_argument("getitem requires at least one argument"); - } - if (py::len(input) == 1) { - try { - ssize_t temp = input[0].cast(); - return getitem(temp); - } - catch (py::cast_error e) { } - try { - py::slice temp = input[0].cast(); - return getitem(temp); - } - catch (py::cast_error e) { } - try { - JaggedArray* temp = input[0].cast(); - throw std::invalid_argument("JaggedArray* argument support not yet implemented"); // TODO - } - catch (py::cast_error e) { } - try { - py::array temp = input[0].cast(); - return getitem(temp); - } - catch (py::cast_error e) { - throw std::invalid_argument("argument type not supported for __getitem__"); - } - } - if (py::len(input) == 2) { - - } - }*/ - - py::object python_getitem(py::tuple input) { - return getitem(input)->unwrap(); - } - py::object tolist() { py::list out; for (ssize_t i = 0; i < len(); i++) { diff --git a/awkward-cpp/awkward/cpp/array/util.h b/awkward-cpp/awkward/cpp/array/util.h index 55a3a096..0c9b4c94 100644 --- a/awkward-cpp/awkward/cpp/array/util.h +++ b/awkward-cpp/awkward/cpp/array/util.h @@ -7,7 +7,6 @@ #include #include #include -#include "CPU_methods.h" namespace py = pybind11; @@ -49,22 +48,31 @@ std::int64_t byteswap(std::int64_t val) { return (val << 32) | ((val >> 32) & 0xFFFFFFFFULL); } +bool isNative(py::array input) { + char ch = input.request().format.at(0); + union { + uint32_t i; + char c[4]; + } bint = { 0x01020304 }; + return ((bint.c[0] == 1 && ch != '<') || (bint.c[0] != 1 && ch != '>')); +} + bool isNativeInt(py::array input) { std::string intList = "qQlLhHbB"; if (intList.find(input.request().format.at(0)) == std::string::npos) { throw std::invalid_argument("argument must be of type int"); } - return isNative(input.request().format.at(0)); + return isNative(input); } template void makeNative(py::array_t input) { - if (isNative(input.request().format.at(0))) { + if (isNative(input)) { return; } py::buffer_info array_info = input.request(); auto array_ptr = (T*)array_info.ptr; - int N = array_info.strides[0] / array_info.itemsize; + int N = array_info.shape[0] / array_info.itemsize; for (ssize_t i = 0; i < array_info.size; i++) { array_ptr[i * N] = byteswap(array_ptr[i * N]); From 03643b8d6460cb01bf80bf3dcced47024b7c3b24 Mon Sep 17 00:00:00 2001 From: EscottC Date: Thu, 11 Jul 2019 23:24:16 -0700 Subject: [PATCH 16/26] offsets2parents, counts2offsets, makeIntNative => CPU --- awkward-cpp/awkward/cpp/array/cpu_methods.h | 240 +++++++++++++++++++ awkward-cpp/awkward/cpp/array/cpu_pybind11.h | 32 +++ awkward-cpp/awkward/cpp/array/jagged.h | 64 ++--- awkward-cpp/awkward/cpp/array/util.h | 102 -------- 4 files changed, 291 insertions(+), 147 deletions(-) create mode 100644 awkward-cpp/awkward/cpp/array/cpu_methods.h create mode 100644 awkward-cpp/awkward/cpp/array/cpu_pybind11.h diff --git a/awkward-cpp/awkward/cpp/array/cpu_methods.h b/awkward-cpp/awkward/cpp/array/cpu_methods.h new file mode 100644 index 00000000..40c5b54c --- /dev/null +++ b/awkward-cpp/awkward/cpp/array/cpu_methods.h @@ -0,0 +1,240 @@ +#ifdef __cplusplus // specifies that this is C, not C++ +extern "C" { +#endif + +#ifndef CPU_METHODS_H // include guard +#define CPU_METHODS_H + +#include + +struct c_array { // this is the C structure of pybind11::buffer_info + void *ptr; + ssize_t itemsize; + ssize_t size; + const char *format; + ssize_t ndim; + const ssize_t *shape; + const ssize_t *strides; +}; + +int byteswap_16bit(int16_t *val) { + *val = *val << 8 | *val >> 8; + return 1; +} + +int byteswap_32bit(int32_t *val) { + *val = ((*val << 8) & 0xFF00FF00) | ((*val >> 8) & 0xFF00FF); + *val = (*val << 16) | (*val >> 16); + return 1; +} + +int byteswap_64bit(int64_t *val) { + *val = ((*val << 8) & 0xFF00FF00FF00FF00ULL) | ((*val >> 8) & 0x00FF00FF00FF00FFULL); + *val = ((*val << 16) & 0xFFFF0000FFFF0000ULL) | ((*val >> 16) & 0x0000FFFF0000FFFFULL); + *val = (*val << 32) | (*val >> 32); + return 1; +} + +int isNative_CPU(struct c_array *input) { + // returns 1 if input is native, or 0 if non-isNative + union { + uint32_t i; + char c[4]; + } bint = { 0x01020304 }; + return ((bint.c[0] == 1 && ((char*)input->format)[0] != '<') + || (bint.c[0] != 1 && ((char*)input->format)[0] != '>')); +} + +int makeNative_16bit(struct c_array *input, ssize_t dim, ssize_t index) { + // to be initially called with makeNative_16bit(input, 0, 0) + if (dim > input->ndim) + return 0; + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + byteswap_16bit(&((int16_t*)input->ptr)[index + i * N]); + return 1; + } + for (ssize_t i = 0; i < input->shape[dim]; i++) + makeNative_16bit(input, dim + 1, index + i * N); + return 1; +} + +int makeNative_32bit(struct c_array *input, ssize_t dim, ssize_t index) { + // to be initially called with makeNative_32bit(input, 0, 0) + if (dim > input->ndim) + return 0; + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + byteswap_32bit(&((int32_t*)input->ptr)[index + i * N]); + return 1; + } + for (ssize_t i = 0; i < input->shape[dim]; i++) + makeNative_32bit(input, dim + 1, index + i * N); + return 1; +} + +int makeNative_64bit(struct c_array *input, ssize_t dim, ssize_t index) { + // to be initially called with makeNative_64bit(input, 0, 0) + if (dim > input->ndim) + return 0; + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + byteswap_64bit(&((int64_t*)input->ptr)[index + i * N]); + return 1; + } + for (ssize_t i = 0; i < input->shape[dim]; i++) + makeNative_64bit(input, dim + 1, index + i * N); + return 1; +} + +int makeNative_CPU(struct c_array *input) { + /* PURPOSE: + - checks if input array is a native array + - if it is non-native, it makes it native + PREREQUISITES: + - can be an array of any dimensionality + */ + if (isNative_CPU(input)) + return 1; + if (input->itemsize == 1) + return 1; + if (input->itemsize == 2) + return makeNative_16bit(input, 0, 0); + if (input->itemsize == 4) + return makeNative_32bit(input, 0, 0); + if (input->itemsize == 8) + return makeNative_64bit(input, 0, 0); + return 0; +} + +int checkInt_CPU(struct c_array *input) { + // returns 1 if it's an int array + char *intList = "qQlLhHbB"; + ssize_t k = 0; + if (input->format[0] == '<' || input->format[0] == '>' || + input->format[0] == '=') + k = 1; + for (ssize_t i = 0; i < 8; i++) + if (intList[i] == input->format[k]) + return 1; + return 0; +} + +int offsets2parents_8bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int8_t*)offsets->ptr)[i * N]) + ((int8_t*)parents->ptr)[j++] = (int8_t)k; + k++; + } + return 1; +} + +int offsets2parents_16bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int16_t*)offsets->ptr)[i * N]) + ((int16_t*)parents->ptr)[j++] = (int16_t)k; + k++; + } + return 1; +} + +int offsets2parents_32bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int32_t*)offsets->ptr)[i * N]) + ((int32_t*)parents->ptr)[j++] = (int32_t)k; + k++; + } + return 1; +} + +int offsets2parents_64bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int64_t*)offsets->ptr)[i * N]) + ((int64_t*)parents->ptr)[j++] = (int64_t)k; + k++; + } + return 1; +} + +int offsets2parents_CPU(struct c_array *offsets, struct c_array *parents) { + /* PURPOSE: + - converts offsets to parents + PREREQUISITES: + - offsets is a non-negative, 1d int array of length > 0 + - parents is a NEW array of size offsets[-1] + - offsets and parents are of the same type + */ + if (offsets->itemsize == 1) + return offsets2parents_8bit(offsets, parents); + if (offsets->itemsize == 2) + return offsets2parents_16bit(offsets, parents); + if (offsets->itemsize == 4) + return offsets2parents_32bit(offsets, parents); + if (offsets->itemsize == 8) + return offsets2parents_64bit(offsets, parents); + return 0; +} + +int counts2offsets_8bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int8_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int8_t*)offsets->ptr)[i + 1] = ((int8_t*)offsets->ptr)[i] + ((int8_t*)counts->ptr)[i * N]; + return 1; +} + +int counts2offsets_16bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int16_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int16_t*)offsets->ptr)[i + 1] = ((int16_t*)offsets->ptr)[i] + ((int16_t*)counts->ptr)[i * N]; + return 1; +} + +int counts2offsets_32bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int32_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int32_t*)offsets->ptr)[i + 1] = ((int32_t*)offsets->ptr)[i] + ((int32_t*)counts->ptr)[i * N]; + return 1; +} + +int counts2offsets_64bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int64_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int64_t*)offsets->ptr)[i + 1] = ((int64_t*)offsets->ptr)[i] + ((int64_t*)counts->ptr)[i * N]; + return 1; +} + +int counts2offsets_CPU(struct c_array *counts, struct c_array *offsets) { + /* PURPOSE: + - converts counts to offsets + PREREQUISITES: + - counts is a non-negative, 1d int array + - offsets is a NEW array of size offsets.size + 1 + - counts and offsets are of the same type + */ + if (counts->itemsize == 1) + return counts2offsets_8bit(counts, offsets); + if (counts->itemsize == 2) + return counts2offsets_16bit(counts, offsets); + if (counts->itemsize == 4) + return counts2offsets_32bit(counts, offsets); + if (counts->itemsize == 8) + return counts2offsets_64bit(counts, offsets); + return 0; +} + +#endif // end include guard + +#ifdef __cplusplus // end C compiler instruction +} +#endif diff --git a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h new file mode 100644 index 00000000..0a557aac --- /dev/null +++ b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include +#include +#include +#include "cpu_methods.h" +#include + +struct c_array py2c(py::array input) { + char format[10]; + strcpy(format, input.request().format.c_str()); + struct c_array out = { + input.request().ptr, + input.request().itemsize, + input.request().size, + format, + input.request().ndim, + &input.request().shape[0], + &input.request().strides[0] + }; + return out; +} + +int makeIntNative_CPU(py::array input) { + if (!checkInt_CPU(&py2c(input))) { + throw std::invalid_argument("Argument must be an int array"); + } + if (!makeNative_CPU(&py2c(input))) { + throw std::exception("Error in cpu_methods.h::makeNative_CPU"); + } + return 1; +} diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index e3bc34c8..2a5b1678 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -1,23 +1,15 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -TODO: -= Expand array types - - std::string - - py::object - - zero-terminated bytes - - raw data -= Add more getitem functions -= Deal with more array characteristics - - Multidimensional arrays -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #pragma once #include #include +#include #include #include #include #include "util.h" #include "any.h" #include "numpytypes.h" +#include "cpu_methods.h" +#include "cpu_pybind11.h" namespace py = pybind11; @@ -59,7 +51,7 @@ class JaggedArray : public AwkwardArray { py::array_t get_starts() { return starts; } void set_starts(py::array starts_) { - makeIntNative(starts_); + makeIntNative_CPU(starts_); starts_ = starts_.cast>(); py::buffer_info starts_info = starts_.request(); if (starts_info.ndim < 1) { @@ -83,7 +75,7 @@ class JaggedArray : public AwkwardArray { py::array_t get_stops() { return stops; } void set_stops(py::array stops_) { - makeIntNative(stops_); + makeIntNative_CPU(stops_); stops_ = stops_.cast>(); py::buffer_info stops_info = stops_.request(); if (stops_info.ndim < 1) { @@ -155,7 +147,7 @@ class JaggedArray : public AwkwardArray { } static JaggedArray* fromoffsets(py::array offsets, AnyArray* content_) { - makeIntNative(offsets); + makeIntNative_CPU(offsets); py::array_t temp = offsets.cast>(); ssize_t length = temp.request().size; if (length < 1) { @@ -309,7 +301,7 @@ class JaggedArray : public AwkwardArray { } static py::array_t offsets2parents(py::array offsets) { - makeIntNative(offsets); + makeIntNative_CPU(offsets); offsets = offsets.cast>(); py::buffer_info offsets_info = offsets.request(); if (offsets_info.size <= 0) { @@ -320,20 +312,10 @@ class JaggedArray : public AwkwardArray { ssize_t parents_length = (ssize_t)offsets_ptr[offsets_info.size - 1]; auto parents = py::array_t(parents_length); - py::buffer_info parents_info = parents.request(); - - auto parents_ptr = (std::int64_t*)parents_info.ptr; - ssize_t j = 0; - ssize_t k = -1; - for (ssize_t i = 0; i < offsets_info.size; i++) { - while (j < (ssize_t)offsets_ptr[i * N]) { - parents_ptr[j] = (std::int64_t)k; - j += 1; - } - k += 1; + if (!offsets2parents_CPU(&py2c(offsets), &py2c(parents))) { + throw std::exception("Error in cpu_methods.h::offsets2parents_CPU"); } - return parents; } @@ -343,31 +325,23 @@ class JaggedArray : public AwkwardArray { } static py::array_t counts2offsets(py::array counts) { - makeIntNative(counts); + makeIntNative_CPU(counts); counts = counts.cast>(); - py::buffer_info counts_info = counts.request(); - auto counts_ptr = (std::int64_t*)counts_info.ptr; - int N = counts_info.strides[0] / counts_info.itemsize; - - ssize_t offsets_length = counts_info.size + 1; - auto offsets = py::array_t(offsets_length); - py::buffer_info offsets_info = offsets.request(); - auto offsets_ptr = (std::int64_t*)offsets_info.ptr; - - offsets_ptr[0] = 0; - for (ssize_t i = 0; i < counts_info.size; i++) { - offsets_ptr[i + 1] = offsets_ptr[i] + (std::int64_t)counts_ptr[i * N]; + auto offsets = py::array_t(counts.request().size + 1); + if (!counts2offsets_CPU(&py2c(counts), &py2c(offsets))) { + throw std::exception("Error in cpu_methods.h::counts2offsets_CPU"); } return offsets; } + static py::array_t python_counts2offsets(py::object countsIter) { py::array counts = countsIter.cast(); return counts2offsets(counts); } static py::array_t startsstops2parents(py::array starts_, py::array stops_) { - makeIntNative(starts_); - makeIntNative(stops_); + makeIntNative_CPU(starts_); + makeIntNative_CPU(stops_); starts_ = starts_.cast>(); stops_ = stops_.cast>(); py::buffer_info starts_info = starts_.request(); @@ -413,7 +387,7 @@ class JaggedArray : public AwkwardArray { } static py::tuple parents2startsstops(py::array parents, std::int64_t length = -1) { - makeIntNative(parents); + makeIntNative_CPU(parents); parents = parents.cast>(); py::buffer_info parents_info = parents.request(); auto parents_ptr = (std::int64_t*)parents_info.ptr; @@ -473,7 +447,7 @@ class JaggedArray : public AwkwardArray { } static py::tuple uniques2offsetsparents(py::array uniques) { - makeIntNative(uniques); + makeIntNative_CPU(uniques); uniques = uniques.cast>(); py::buffer_info uniques_info = uniques.request(); auto uniques_ptr = (std::int64_t*)uniques_info.ptr; @@ -650,7 +624,7 @@ class JaggedArray : public AwkwardArray { } JaggedArray* intarray_getitem(py::array input) { - makeIntNative(input); + makeIntNative_CPU(input); input = input.cast>(); py::buffer_info array_info = input.request(); auto array_ptr = (std::int64_t*)array_info.ptr; diff --git a/awkward-cpp/awkward/cpp/array/util.h b/awkward-cpp/awkward/cpp/array/util.h index 0c9b4c94..ae69d981 100644 --- a/awkward-cpp/awkward/cpp/array/util.h +++ b/awkward-cpp/awkward/cpp/array/util.h @@ -10,108 +10,6 @@ namespace py = pybind11; -std::uint8_t byteswap(std::uint8_t val) { - return val; -} - -std::int8_t byteswap(std::int8_t val) { - return val; -} - -std::uint16_t byteswap(std::uint16_t val) { - return (val << 8) | (val >> 8); -} - -std::int16_t byteswap(std::int16_t val) { - return (val << 8) | ((val >> 8) & 0xFF); -} - -std::uint32_t byteswap(std::uint32_t val) { - val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF); - return (val << 16) | (val >> 16); -} - -std::int32_t byteswap(std::int32_t val) { - val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF); - return (val << 16) | ((val >> 16) & 0xFFFF); -} - -std::uint64_t byteswap(std::uint64_t val) { - val = ((val << 8) & 0xFF00FF00FF00FF00ULL) | ((val >> 8) & 0x00FF00FF00FF00FFULL); - val = ((val << 16) & 0xFFFF0000FFFF0000ULL) | ((val >> 16) & 0x0000FFFF0000FFFFULL); - return (val << 32) | (val >> 32); -} - -std::int64_t byteswap(std::int64_t val) { - val = ((val << 8) & 0xFF00FF00FF00FF00ULL) | ((val >> 8) & 0x00FF00FF00FF00FFULL); - val = ((val << 16) & 0xFFFF0000FFFF0000ULL) | ((val >> 16) & 0x0000FFFF0000FFFFULL); - return (val << 32) | ((val >> 32) & 0xFFFFFFFFULL); -} - -bool isNative(py::array input) { - char ch = input.request().format.at(0); - union { - uint32_t i; - char c[4]; - } bint = { 0x01020304 }; - return ((bint.c[0] == 1 && ch != '<') || (bint.c[0] != 1 && ch != '>')); -} - -bool isNativeInt(py::array input) { - std::string intList = "qQlLhHbB"; - if (intList.find(input.request().format.at(0)) == std::string::npos) { - throw std::invalid_argument("argument must be of type int"); - } - return isNative(input); -} - -template -void makeNative(py::array_t input) { - if (isNative(input)) { - return; - } - py::buffer_info array_info = input.request(); - auto array_ptr = (T*)array_info.ptr; - int N = array_info.shape[0] / array_info.itemsize; - - for (ssize_t i = 0; i < array_info.size; i++) { - array_ptr[i * N] = byteswap(array_ptr[i * N]); - } - return; -} - -void makeIntNative(py::array input) { - if (isNativeInt(input)) { - return; - } - char code = input.request().format.at(1); - if (code == 'q') { - makeNative(input.cast>()); - return; - } - if (code == 'Q') { - makeNative(input.cast>()); - return; - } - if (code == 'l') { - makeNative(input.cast>()); - return; - } - if (code == 'L') { - makeNative(input.cast>()); - return; - } - if (code == 'h') { - makeNative(input.cast>()); - return; - } - if (code == 'H') { - makeNative(input.cast>()); - return; - } - throw std::invalid_argument("argument must be of type int"); -} - template py::array_t slice_numpy(py::array_t input, ssize_t start, ssize_t length, ssize_t step = 1) { ssize_t arrayLen = input.request().size; From a1224de5ee6b41b3edc3aa58123b87a1d0db991d Mon Sep 17 00:00:00 2001 From: EscottC Date: Fri, 12 Jul 2019 11:56:16 -0700 Subject: [PATCH 17/26] hopefully fixing temp address error --- awkward-cpp/awkward/cpp/array/cpu_methods.h | 166 +++++++++---------- awkward-cpp/awkward/cpp/array/cpu_pybind11.h | 8 +- awkward-cpp/awkward/cpp/array/jagged.h | 6 +- 3 files changed, 91 insertions(+), 89 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/cpu_methods.h b/awkward-cpp/awkward/cpp/array/cpu_methods.h index 40c5b54c..39a6d65e 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_methods.h +++ b/awkward-cpp/awkward/cpp/array/cpu_methods.h @@ -35,62 +35,62 @@ int byteswap_64bit(int64_t *val) { return 1; } -int isNative_CPU(struct c_array *input) { +int isNative_CPU(struct c_array input) { // returns 1 if input is native, or 0 if non-isNative union { uint32_t i; char c[4]; } bint = { 0x01020304 }; - return ((bint.c[0] == 1 && ((char*)input->format)[0] != '<') - || (bint.c[0] != 1 && ((char*)input->format)[0] != '>')); + return ((bint.c[0] == 1 && ((char*)input.format)[0] != '<') + || (bint.c[0] != 1 && ((char*)input.format)[0] != '>')); } -int makeNative_16bit(struct c_array *input, ssize_t dim, ssize_t index) { +int makeNative_16bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_16bit(input, 0, 0) - if (dim > input->ndim) + if (dim > input.ndim) return 0; - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - byteswap_16bit(&((int16_t*)input->ptr)[index + i * N]); + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + byteswap_16bit(&((int16_t*)input.ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) makeNative_16bit(input, dim + 1, index + i * N); return 1; } -int makeNative_32bit(struct c_array *input, ssize_t dim, ssize_t index) { +int makeNative_32bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_32bit(input, 0, 0) - if (dim > input->ndim) + if (dim > input.ndim) return 0; - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - byteswap_32bit(&((int32_t*)input->ptr)[index + i * N]); + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + byteswap_32bit(&((int32_t*)input.ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) makeNative_32bit(input, dim + 1, index + i * N); return 1; } -int makeNative_64bit(struct c_array *input, ssize_t dim, ssize_t index) { +int makeNative_64bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_64bit(input, 0, 0) - if (dim > input->ndim) + if (dim > input.ndim) return 0; - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - byteswap_64bit(&((int64_t*)input->ptr)[index + i * N]); + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + byteswap_64bit(&((int64_t*)input.ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) makeNative_64bit(input, dim + 1, index + i * N); return 1; } -int makeNative_CPU(struct c_array *input) { +int makeNative_CPU(struct c_array input) { /* PURPOSE: - checks if input array is a native array - if it is non-native, it makes it native @@ -99,71 +99,71 @@ int makeNative_CPU(struct c_array *input) { */ if (isNative_CPU(input)) return 1; - if (input->itemsize == 1) + if (input.itemsize == 1) return 1; - if (input->itemsize == 2) + if (input.itemsize == 2) return makeNative_16bit(input, 0, 0); - if (input->itemsize == 4) + if (input.itemsize == 4) return makeNative_32bit(input, 0, 0); - if (input->itemsize == 8) + if (input.itemsize == 8) return makeNative_64bit(input, 0, 0); return 0; } -int checkInt_CPU(struct c_array *input) { +int checkInt_CPU(struct c_array input) { // returns 1 if it's an int array char *intList = "qQlLhHbB"; ssize_t k = 0; - if (input->format[0] == '<' || input->format[0] == '>' || - input->format[0] == '=') + if (input.format[0] == '<' || input.format[0] == '>' || + input.format[0] == '=') k = 1; for (ssize_t i = 0; i < 8; i++) - if (intList[i] == input->format[k]) + if (intList[i] == input.format[k]) return 1; return 0; } -int offsets2parents_8bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int8_t*)offsets->ptr)[i * N]) - ((int8_t*)parents->ptr)[j++] = (int8_t)k; +int offsets2parents_8bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int8_t*)offsets.ptr)[i * N]) + ((int8_t*)parents.ptr)[j++] = (int8_t)k; k++; } return 1; } -int offsets2parents_16bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int16_t*)offsets->ptr)[i * N]) - ((int16_t*)parents->ptr)[j++] = (int16_t)k; +int offsets2parents_16bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int16_t*)offsets.ptr)[i * N]) + ((int16_t*)parents.ptr)[j++] = (int16_t)k; k++; } return 1; } -int offsets2parents_32bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int32_t*)offsets->ptr)[i * N]) - ((int32_t*)parents->ptr)[j++] = (int32_t)k; +int offsets2parents_32bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int32_t*)offsets.ptr)[i * N]) + ((int32_t*)parents.ptr)[j++] = (int32_t)k; k++; } return 1; } -int offsets2parents_64bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int64_t*)offsets->ptr)[i * N]) - ((int64_t*)parents->ptr)[j++] = (int64_t)k; +int offsets2parents_64bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int64_t*)offsets.ptr)[i * N]) + ((int64_t*)parents.ptr)[j++] = (int64_t)k; k++; } return 1; } -int offsets2parents_CPU(struct c_array *offsets, struct c_array *parents) { +int offsets2parents_CPU(struct c_array offsets, struct c_array parents) { /* PURPOSE: - converts offsets to parents PREREQUISITES: @@ -171,50 +171,50 @@ int offsets2parents_CPU(struct c_array *offsets, struct c_array *parents) { - parents is a NEW array of size offsets[-1] - offsets and parents are of the same type */ - if (offsets->itemsize == 1) + if (offsets.itemsize == 1) return offsets2parents_8bit(offsets, parents); - if (offsets->itemsize == 2) + if (offsets.itemsize == 2) return offsets2parents_16bit(offsets, parents); - if (offsets->itemsize == 4) + if (offsets.itemsize == 4) return offsets2parents_32bit(offsets, parents); - if (offsets->itemsize == 8) + if (offsets.itemsize == 8) return offsets2parents_64bit(offsets, parents); return 0; } -int counts2offsets_8bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int8_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int8_t*)offsets->ptr)[i + 1] = ((int8_t*)offsets->ptr)[i] + ((int8_t*)counts->ptr)[i * N]; +int counts2offsets_8bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int8_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int8_t*)offsets.ptr)[i + 1] = ((int8_t*)offsets.ptr)[i] + ((int8_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_16bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int16_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int16_t*)offsets->ptr)[i + 1] = ((int16_t*)offsets->ptr)[i] + ((int16_t*)counts->ptr)[i * N]; +int counts2offsets_16bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int16_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int16_t*)offsets.ptr)[i + 1] = ((int16_t*)offsets.ptr)[i] + ((int16_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_32bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int32_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int32_t*)offsets->ptr)[i + 1] = ((int32_t*)offsets->ptr)[i] + ((int32_t*)counts->ptr)[i * N]; +int counts2offsets_32bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int32_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int32_t*)offsets.ptr)[i + 1] = ((int32_t*)offsets.ptr)[i] + ((int32_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_64bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int64_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int64_t*)offsets->ptr)[i + 1] = ((int64_t*)offsets->ptr)[i] + ((int64_t*)counts->ptr)[i * N]; +int counts2offsets_64bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int64_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int64_t*)offsets.ptr)[i + 1] = ((int64_t*)offsets.ptr)[i] + ((int64_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_CPU(struct c_array *counts, struct c_array *offsets) { +int counts2offsets_CPU(struct c_array counts, struct c_array offsets) { /* PURPOSE: - converts counts to offsets PREREQUISITES: @@ -222,13 +222,13 @@ int counts2offsets_CPU(struct c_array *counts, struct c_array *offsets) { - offsets is a NEW array of size offsets.size + 1 - counts and offsets are of the same type */ - if (counts->itemsize == 1) + if (counts.itemsize == 1) return counts2offsets_8bit(counts, offsets); - if (counts->itemsize == 2) + if (counts.itemsize == 2) return counts2offsets_16bit(counts, offsets); - if (counts->itemsize == 4) + if (counts.itemsize == 4) return counts2offsets_32bit(counts, offsets); - if (counts->itemsize == 8) + if (counts.itemsize == 8) return counts2offsets_64bit(counts, offsets); return 0; } diff --git a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h index 0a557aac..3c22b760 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h +++ b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h @@ -6,8 +6,10 @@ #include "cpu_methods.h" #include +namespace py = pybind11; + struct c_array py2c(py::array input) { - char format[10]; + char format[8]; strcpy(format, input.request().format.c_str()); struct c_array out = { input.request().ptr, @@ -22,10 +24,10 @@ struct c_array py2c(py::array input) { } int makeIntNative_CPU(py::array input) { - if (!checkInt_CPU(&py2c(input))) { + if (!checkInt_CPU(py2c(input))) { throw std::invalid_argument("Argument must be an int array"); } - if (!makeNative_CPU(&py2c(input))) { + if (!makeNative_CPU(py2c(input))) { throw std::exception("Error in cpu_methods.h::makeNative_CPU"); } return 1; diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 2a5b1678..ef6ecce9 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -310,10 +310,10 @@ class JaggedArray : public AwkwardArray { auto offsets_ptr = (std::int64_t*)offsets_info.ptr; int N = offsets_info.strides[0] / offsets_info.itemsize; - ssize_t parents_length = (ssize_t)offsets_ptr[offsets_info.size - 1]; + ssize_t parents_length = (ssize_t)offsets_ptr[(offsets_info.size - 1) * N]; auto parents = py::array_t(parents_length); - if (!offsets2parents_CPU(&py2c(offsets), &py2c(parents))) { + if (!offsets2parents_CPU(py2c(offsets), py2c(parents))) { throw std::exception("Error in cpu_methods.h::offsets2parents_CPU"); } return parents; @@ -328,7 +328,7 @@ class JaggedArray : public AwkwardArray { makeIntNative_CPU(counts); counts = counts.cast>(); auto offsets = py::array_t(counts.request().size + 1); - if (!counts2offsets_CPU(&py2c(counts), &py2c(offsets))) { + if (!counts2offsets_CPU(py2c(counts), py2c(offsets))) { throw std::exception("Error in cpu_methods.h::counts2offsets_CPU"); } return offsets; From 2ef71a7d81215306d99e674f19873dc8747ea3f1 Mon Sep 17 00:00:00 2001 From: EscottC Date: Fri, 12 Jul 2019 13:33:12 -0700 Subject: [PATCH 18/26] startsstops2parents_CPU and getMax_CPU --- awkward-cpp/awkward/cpp/array/cpu_methods.h | 171 ++++++++++++++++++- awkward-cpp/awkward/cpp/array/cpu_pybind11.h | 28 +++ awkward-cpp/awkward/cpp/array/jagged.h | 35 +--- 3 files changed, 196 insertions(+), 38 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/cpu_methods.h b/awkward-cpp/awkward/cpp/array/cpu_methods.h index 39a6d65e..4f258c24 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_methods.h +++ b/awkward-cpp/awkward/cpp/array/cpu_methods.h @@ -41,16 +41,16 @@ int isNative_CPU(struct c_array input) { uint32_t i; char c[4]; } bint = { 0x01020304 }; - return ((bint.c[0] == 1 && ((char*)input.format)[0] != '<') - || (bint.c[0] != 1 && ((char*)input.format)[0] != '>')); + return ((bint.c[0] == 1 && input.format[0] != '<') + || (bint.c[0] != 1 && input.format[0] != '>')); } int makeNative_16bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_16bit(input, 0, 0) - if (dim > input.ndim) + if (dim > input.ndim - 1) return 0; ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim) { + if (dim == input.ndim - 1) { for (ssize_t i = 0; i < input.shape[dim]; i++) byteswap_16bit(&((int16_t*)input.ptr)[index + i * N]); return 1; @@ -62,10 +62,10 @@ int makeNative_16bit(struct c_array input, ssize_t dim, ssize_t index) { int makeNative_32bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_32bit(input, 0, 0) - if (dim > input.ndim) + if (dim > input.ndim - 1) return 0; ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim) { + if (dim == input.ndim - 1) { for (ssize_t i = 0; i < input.shape[dim]; i++) byteswap_32bit(&((int32_t*)input.ptr)[index + i * N]); return 1; @@ -77,10 +77,10 @@ int makeNative_32bit(struct c_array input, ssize_t dim, ssize_t index) { int makeNative_64bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_64bit(input, 0, 0) - if (dim > input.ndim) + if (dim > input.ndim - 1) return 0; ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim) { + if (dim == input.ndim - 1) { for (ssize_t i = 0; i < input.shape[dim]; i++) byteswap_64bit(&((int64_t*)input.ptr)[index + i * N]); return 1; @@ -123,6 +123,98 @@ int checkInt_CPU(struct c_array input) { return 0; } +int getMax_8bit(struct c_array input, ssize_t dim, ssize_t index, int8_t *max) { + // to be initially called with getMax_8bit(input, 0, 0, *0) + if (dim > input.ndim - 1) + return 0; + if (dim == 0) { + if (input.shape[0] == 0) { + *max = 0; + return 1; + } + *max = ((int8_t*)input.ptr)[0]; + } + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int8_t*)input.ptr)[index + i * N] > *max) + *max = ((int8_t*)input.ptr)[index + i * N]; + return 1; + } + for (ssize_t i = 0; i < input.shape[dim]; i++) + getMax_8bit(input, dim + 1, index + i * N, max); + return 1; +} + +int getMax_16bit(struct c_array input, ssize_t dim, ssize_t index, int16_t *max) { + // to be initially called with getMax_16bit(input, 0, 0, *0) + if (dim > input.ndim - 1) + return 0; + if (dim == 0) { + if (input.shape[0] == 0) { + *max = 0; + return 1; + } + *max = ((int16_t*)input.ptr)[0]; + } + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int16_t*)input.ptr)[index + i * N] > *max) + *max = ((int16_t*)input.ptr)[index + i * N]; + return 1; + } + for (ssize_t i = 0; i < input.shape[dim]; i++) + getMax_16bit(input, dim + 1, index + i * N, max); + return 1; +} + +int getMax_32bit(struct c_array input, ssize_t dim, ssize_t index, int32_t *max) { + // to be initially called with getMax_32bit(input, 0, 0, *0) + if (dim > input.ndim - 1) + return 0; + if (dim == 0) { + if (input.shape[0] == 0) { + *max = 0; + return 1; + } + *max = ((int32_t*)input.ptr)[0]; + } + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int32_t*)input.ptr)[index + i * N] > *max) + *max = ((int32_t*)input.ptr)[index + i * N]; + return 1; + } + for (ssize_t i = 0; i < input.shape[dim]; i++) + getMax_32bit(input, dim + 1, index + i * N, max); + return 1; +} + +int getMax_64bit(struct c_array input, ssize_t dim, ssize_t index, int64_t *max) { + // to be initially called with getMax_64bit(input, 0, 0, *0) + if (dim > input.ndim - 1) + return 0; + if (dim == 0) { + if (input.shape[0] == 0) { + *max = 0; + return 1; + } + *max = ((int64_t*)input.ptr)[0]; + } + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int64_t*)input.ptr)[index + i * N] > *max) + *max = ((int64_t*)input.ptr)[index + i * N]; + return 1; + } + for (ssize_t i = 0; i < input.shape[dim]; i++) + getMax_64bit(input, dim + 1, index + i * N, max); + return 1; +} + int offsets2parents_8bit(struct c_array offsets, struct c_array parents) { ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; for (ssize_t i = 0; i < offsets.size; i++) { @@ -221,6 +313,7 @@ int counts2offsets_CPU(struct c_array counts, struct c_array offsets) { - counts is a non-negative, 1d int array - offsets is a NEW array of size offsets.size + 1 - counts and offsets are of the same type + - note: for >1d counts, flatten it first */ if (counts.itemsize == 1) return counts2offsets_8bit(counts, offsets); @@ -233,6 +326,68 @@ int counts2offsets_CPU(struct c_array counts, struct c_array offsets) { return 0; } +int startsstops2parents_8bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int8_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int8_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int8_t*)stops.ptr)[i * N_sto]; j++) + ((int8_t*)parents.ptr)[j] = (int8_t)i; + return 1; +} + +int startsstops2parents_16bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int16_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int16_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int16_t*)stops.ptr)[i * N_sto]; j++) + ((int16_t*)parents.ptr)[j] = (int16_t)i; + return 1; +} + +int startsstops2parents_32bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int32_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int32_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int32_t*)stops.ptr)[i * N_sto]; j++) + ((int32_t*)parents.ptr)[j] = (int32_t)i; + return 1; +} + +int startsstops2parents_64bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int64_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int64_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int64_t*)stops.ptr)[i * N_sto]; j++) + ((int64_t*)parents.ptr)[j] = (int64_t)i; + return 1; +} + +int startsstops2parents_CPU(struct c_array starts, struct c_array stops, struct c_array parents) { + /* PURPOSE: + - converts starts and stops to parents + PREREQUISITES: + - starts and stops are non-negative 1d int arrays + - starts must be less than or equal to stops + - starts must be of equal or shorter length than stops + - parents is a NEW array of length stops.maximum + - starts, stops, and parents are of the same type + - note: for >1d starts/stops, flatten them first + */ + if (starts.itemsize == 1) + return startsstops2parents_8bit(starts, stops, parents); + if (starts.itemsize == 2) + return startsstops2parents_16bit(starts, stops, parents); + if (starts.itemsize == 4) + return startsstops2parents_32bit(starts, stops, parents); + if (starts.itemsize == 8) + return startsstops2parents_64bit(starts, stops, parents); + return 0; +} + #endif // end include guard #ifdef __cplusplus // end C compiler instruction diff --git a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h index 3c22b760..6c0b396e 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h +++ b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h @@ -32,3 +32,31 @@ int makeIntNative_CPU(py::array input) { } return 1; } + +int getMax_CPU(py::array input, std::int8_t* max) { + if (!getMax_8bit(py2c(input), 0, 0, max)) { + throw std::exception("Error in cpu_methods.h::getMax_8bit"); + } + return 1; +} + +int getMax_CPU(py::array input, std::int16_t* max) { + if (!getMax_16bit(py2c(input), 0, 0, max)) { + throw std::exception("Error in cpu_methods.h::getMax_16bit"); + } + return 1; +} + +int getMax_CPU(py::array input, std::int32_t* max) { + if (!getMax_32bit(py2c(input), 0, 0, max)) { + throw std::exception("Error in cpu_methods.h::getMax_32bit"); + } + return 1; +} + +int getMax_CPU(py::array input, std::int64_t* max) { + if (!getMax_64bit(py2c(input), 0, 0, max)) { + throw std::exception("Error in cpu_methods.h::getMax_64bit"); + } + return 1; +} diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index ef6ecce9..2fb054c4 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -344,39 +344,14 @@ class JaggedArray : public AwkwardArray { makeIntNative_CPU(stops_); starts_ = starts_.cast>(); stops_ = stops_.cast>(); - py::buffer_info starts_info = starts_.request(); - auto starts_ptr = (std::int64_t*)starts_info.ptr; - int N_starts = starts_info.strides[0] / starts_info.itemsize; - py::buffer_info stops_info = stops_.request(); - auto stops_ptr = (std::int64_t*)stops_info.ptr; - int N_stops = stops_info.strides[0] / stops_info.itemsize; - - ssize_t max; - if (stops_info.size < 1) { - max = 0; - } - else { - max = (ssize_t)stops_ptr[0]; - for (ssize_t i = 1; i < stops_info.size; i++) { - if ((ssize_t)stops_ptr[i * N_stops] > max) { - max = (ssize_t)stops_ptr[i * N_stops]; - } - } - } - auto parents = py::array_t(max); - py::buffer_info parents_info = parents.request(); - auto parents_ptr = (std::int64_t*)parents_info.ptr; - for (ssize_t i = 0; i < max; i++) { - parents_ptr[i] = -1; - } + std::int64_t max = 0; + getMax_CPU(stops_, &max); + auto parents = py::array_t((ssize_t)max); - for (ssize_t i = 0; i < starts_info.size; i++) { - for (ssize_t j = (ssize_t)starts_ptr[i * N_starts]; j < (ssize_t)stops_ptr[i * N_stops]; j++) { - parents_ptr[j] = (std::int64_t)i; - } + if (!startsstops2parents_CPU(py2c(starts_), py2c(stops_), py2c(parents))) { + throw std::exception("Error in cpu_methods.h::startsstops2parents_CPU"); } - return parents; } From aaccfee993e028b0a1df966353931c675ef7f4fb Mon Sep 17 00:00:00 2001 From: EscottC Date: Fri, 12 Jul 2019 13:41:29 -0700 Subject: [PATCH 19/26] fixing travis errors attempt #1 --- awkward-cpp/awkward/cpp/array/cpu_methods.h | 2 +- awkward-cpp/awkward/cpp/array/cpu_pybind11.h | 10 +++++----- awkward-cpp/awkward/cpp/array/jagged.h | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/cpu_methods.h b/awkward-cpp/awkward/cpp/array/cpu_methods.h index 4f258c24..f9c2d317 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_methods.h +++ b/awkward-cpp/awkward/cpp/array/cpu_methods.h @@ -112,7 +112,7 @@ int makeNative_CPU(struct c_array input) { int checkInt_CPU(struct c_array input) { // returns 1 if it's an int array - char *intList = "qQlLhHbB"; + char intList[] = "qQlLhHbB"; ssize_t k = 0; if (input.format[0] == '<' || input.format[0] == '>' || input.format[0] == '=') diff --git a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h index 6c0b396e..b370514f 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h +++ b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h @@ -28,35 +28,35 @@ int makeIntNative_CPU(py::array input) { throw std::invalid_argument("Argument must be an int array"); } if (!makeNative_CPU(py2c(input))) { - throw std::exception("Error in cpu_methods.h::makeNative_CPU"); + throw std::invalid_argument("Error in cpu_methods.h::makeNative_CPU"); } return 1; } int getMax_CPU(py::array input, std::int8_t* max) { if (!getMax_8bit(py2c(input), 0, 0, max)) { - throw std::exception("Error in cpu_methods.h::getMax_8bit"); + throw std::invalid_argument("Error in cpu_methods.h::getMax_8bit"); } return 1; } int getMax_CPU(py::array input, std::int16_t* max) { if (!getMax_16bit(py2c(input), 0, 0, max)) { - throw std::exception("Error in cpu_methods.h::getMax_16bit"); + throw std::invalid_argument("Error in cpu_methods.h::getMax_16bit"); } return 1; } int getMax_CPU(py::array input, std::int32_t* max) { if (!getMax_32bit(py2c(input), 0, 0, max)) { - throw std::exception("Error in cpu_methods.h::getMax_32bit"); + throw std::invalid_argument("Error in cpu_methods.h::getMax_32bit"); } return 1; } int getMax_CPU(py::array input, std::int64_t* max) { if (!getMax_64bit(py2c(input), 0, 0, max)) { - throw std::exception("Error in cpu_methods.h::getMax_64bit"); + throw std::invalid_argument("Error in cpu_methods.h::getMax_64bit"); } return 1; } diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 2fb054c4..b3996a8d 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -314,7 +314,7 @@ class JaggedArray : public AwkwardArray { auto parents = py::array_t(parents_length); if (!offsets2parents_CPU(py2c(offsets), py2c(parents))) { - throw std::exception("Error in cpu_methods.h::offsets2parents_CPU"); + throw std::invalid_argument("Error in cpu_methods.h::offsets2parents_CPU"); } return parents; } @@ -329,7 +329,7 @@ class JaggedArray : public AwkwardArray { counts = counts.cast>(); auto offsets = py::array_t(counts.request().size + 1); if (!counts2offsets_CPU(py2c(counts), py2c(offsets))) { - throw std::exception("Error in cpu_methods.h::counts2offsets_CPU"); + throw std::invalid_argument("Error in cpu_methods.h::counts2offsets_CPU"); } return offsets; } @@ -350,7 +350,7 @@ class JaggedArray : public AwkwardArray { auto parents = py::array_t((ssize_t)max); if (!startsstops2parents_CPU(py2c(starts_), py2c(stops_), py2c(parents))) { - throw std::exception("Error in cpu_methods.h::startsstops2parents_CPU"); + throw std::invalid_argument("Error in cpu_methods.h::startsstops2parents_CPU"); } return parents; } From 5c66059fb2ffd5e7c2db5cafa4fbadde0a6ad27a Mon Sep 17 00:00:00 2001 From: EscottC Date: Fri, 12 Jul 2019 14:27:13 -0700 Subject: [PATCH 20/26] fixing travis errors attempt #2 --- awkward-cpp/awkward/cpp/array/cpu_methods.h | 316 +++++++++---------- awkward-cpp/awkward/cpp/array/cpu_pybind11.h | 12 +- awkward-cpp/awkward/cpp/array/jagged.h | 6 +- 3 files changed, 167 insertions(+), 167 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/cpu_methods.h b/awkward-cpp/awkward/cpp/array/cpu_methods.h index f9c2d317..3606459e 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_methods.h +++ b/awkward-cpp/awkward/cpp/array/cpu_methods.h @@ -35,62 +35,62 @@ int byteswap_64bit(int64_t *val) { return 1; } -int isNative_CPU(struct c_array input) { +int isNative_CPU(struct c_array *input) { // returns 1 if input is native, or 0 if non-isNative union { uint32_t i; char c[4]; } bint = { 0x01020304 }; - return ((bint.c[0] == 1 && input.format[0] != '<') - || (bint.c[0] != 1 && input.format[0] != '>')); + return ((bint.c[0] == 1 && input->format[0] != '<') + || (bint.c[0] != 1 && input->format[0] != '>')); } -int makeNative_16bit(struct c_array input, ssize_t dim, ssize_t index) { +int makeNative_16bit(struct c_array *input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_16bit(input, 0, 0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - byteswap_16bit(&((int16_t*)input.ptr)[index + i * N]); + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + byteswap_16bit(&((int16_t*)input->ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) makeNative_16bit(input, dim + 1, index + i * N); return 1; } -int makeNative_32bit(struct c_array input, ssize_t dim, ssize_t index) { +int makeNative_32bit(struct c_array *input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_32bit(input, 0, 0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - byteswap_32bit(&((int32_t*)input.ptr)[index + i * N]); + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + byteswap_32bit(&((int32_t*)input->ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) makeNative_32bit(input, dim + 1, index + i * N); return 1; } -int makeNative_64bit(struct c_array input, ssize_t dim, ssize_t index) { +int makeNative_64bit(struct c_array *input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_64bit(input, 0, 0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - byteswap_64bit(&((int64_t*)input.ptr)[index + i * N]); + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + byteswap_64bit(&((int64_t*)input->ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) makeNative_64bit(input, dim + 1, index + i * N); return 1; } -int makeNative_CPU(struct c_array input) { +int makeNative_CPU(struct c_array *input) { /* PURPOSE: - checks if input array is a native array - if it is non-native, it makes it native @@ -99,163 +99,163 @@ int makeNative_CPU(struct c_array input) { */ if (isNative_CPU(input)) return 1; - if (input.itemsize == 1) + if (input->itemsize == 1) return 1; - if (input.itemsize == 2) + if (input->itemsize == 2) return makeNative_16bit(input, 0, 0); - if (input.itemsize == 4) + if (input->itemsize == 4) return makeNative_32bit(input, 0, 0); - if (input.itemsize == 8) + if (input->itemsize == 8) return makeNative_64bit(input, 0, 0); return 0; } -int checkInt_CPU(struct c_array input) { +int checkInt_CPU(struct c_array *input) { // returns 1 if it's an int array char intList[] = "qQlLhHbB"; ssize_t k = 0; - if (input.format[0] == '<' || input.format[0] == '>' || - input.format[0] == '=') + if (input->format[0] == '<' || input->format[0] == '>' || + input->format[0] == '=') k = 1; for (ssize_t i = 0; i < 8; i++) - if (intList[i] == input.format[k]) + if (intList[i] == input->format[k]) return 1; return 0; } -int getMax_8bit(struct c_array input, ssize_t dim, ssize_t index, int8_t *max) { +int getMax_8bit(struct c_array *input, ssize_t dim, ssize_t index, int8_t *max) { // to be initially called with getMax_8bit(input, 0, 0, *0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; if (dim == 0) { - if (input.shape[0] == 0) { + if (input->shape[0] == 0) { *max = 0; return 1; } - *max = ((int8_t*)input.ptr)[0]; + *max = ((int8_t*)input->ptr)[0]; } - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - if (((int8_t*)input.ptr)[index + i * N] > *max) - *max = ((int8_t*)input.ptr)[index + i * N]; + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + if (((int8_t*)input->ptr)[index + i * N] > *max) + *max = ((int8_t*)input->ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) getMax_8bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_16bit(struct c_array input, ssize_t dim, ssize_t index, int16_t *max) { +int getMax_16bit(struct c_array *input, ssize_t dim, ssize_t index, int16_t *max) { // to be initially called with getMax_16bit(input, 0, 0, *0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; if (dim == 0) { - if (input.shape[0] == 0) { + if (input->shape[0] == 0) { *max = 0; return 1; } - *max = ((int16_t*)input.ptr)[0]; + *max = ((int16_t*)input->ptr)[0]; } - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - if (((int16_t*)input.ptr)[index + i * N] > *max) - *max = ((int16_t*)input.ptr)[index + i * N]; + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + if (((int16_t*)input->ptr)[index + i * N] > *max) + *max = ((int16_t*)input->ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) getMax_16bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_32bit(struct c_array input, ssize_t dim, ssize_t index, int32_t *max) { +int getMax_32bit(struct c_array *input, ssize_t dim, ssize_t index, int32_t *max) { // to be initially called with getMax_32bit(input, 0, 0, *0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; if (dim == 0) { - if (input.shape[0] == 0) { + if (input->shape[0] == 0) { *max = 0; return 1; } - *max = ((int32_t*)input.ptr)[0]; + *max = ((int32_t*)input->ptr)[0]; } - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - if (((int32_t*)input.ptr)[index + i * N] > *max) - *max = ((int32_t*)input.ptr)[index + i * N]; + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + if (((int32_t*)input->ptr)[index + i * N] > *max) + *max = ((int32_t*)input->ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) getMax_32bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_64bit(struct c_array input, ssize_t dim, ssize_t index, int64_t *max) { +int getMax_64bit(struct c_array *input, ssize_t dim, ssize_t index, int64_t *max) { // to be initially called with getMax_64bit(input, 0, 0, *0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; if (dim == 0) { - if (input.shape[0] == 0) { + if (input->shape[0] == 0) { *max = 0; return 1; } - *max = ((int64_t*)input.ptr)[0]; + *max = ((int64_t*)input->ptr)[0]; } - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - if (((int64_t*)input.ptr)[index + i * N] > *max) - *max = ((int64_t*)input.ptr)[index + i * N]; + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + if (((int64_t*)input->ptr)[index + i * N] > *max) + *max = ((int64_t*)input->ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) getMax_64bit(input, dim + 1, index + i * N, max); return 1; } -int offsets2parents_8bit(struct c_array offsets, struct c_array parents) { - ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets.size; i++) { - while (j < (ssize_t)((int8_t*)offsets.ptr)[i * N]) - ((int8_t*)parents.ptr)[j++] = (int8_t)k; +int offsets2parents_8bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int8_t*)offsets->ptr)[i * N]) + ((int8_t*)parents->ptr)[j++] = (int8_t)k; k++; } return 1; } -int offsets2parents_16bit(struct c_array offsets, struct c_array parents) { - ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets.size; i++) { - while (j < (ssize_t)((int16_t*)offsets.ptr)[i * N]) - ((int16_t*)parents.ptr)[j++] = (int16_t)k; +int offsets2parents_16bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int16_t*)offsets->ptr)[i * N]) + ((int16_t*)parents->ptr)[j++] = (int16_t)k; k++; } return 1; } -int offsets2parents_32bit(struct c_array offsets, struct c_array parents) { - ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets.size; i++) { - while (j < (ssize_t)((int32_t*)offsets.ptr)[i * N]) - ((int32_t*)parents.ptr)[j++] = (int32_t)k; +int offsets2parents_32bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int32_t*)offsets->ptr)[i * N]) + ((int32_t*)parents->ptr)[j++] = (int32_t)k; k++; } return 1; } -int offsets2parents_64bit(struct c_array offsets, struct c_array parents) { - ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets.size; i++) { - while (j < (ssize_t)((int64_t*)offsets.ptr)[i * N]) - ((int64_t*)parents.ptr)[j++] = (int64_t)k; +int offsets2parents_64bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int64_t*)offsets->ptr)[i * N]) + ((int64_t*)parents->ptr)[j++] = (int64_t)k; k++; } return 1; } -int offsets2parents_CPU(struct c_array offsets, struct c_array parents) { +int offsets2parents_CPU(struct c_array *offsets, struct c_array *parents) { /* PURPOSE: - converts offsets to parents PREREQUISITES: @@ -263,127 +263,127 @@ int offsets2parents_CPU(struct c_array offsets, struct c_array parents) { - parents is a NEW array of size offsets[-1] - offsets and parents are of the same type */ - if (offsets.itemsize == 1) + if (offsets->itemsize == 1) return offsets2parents_8bit(offsets, parents); - if (offsets.itemsize == 2) + if (offsets->itemsize == 2) return offsets2parents_16bit(offsets, parents); - if (offsets.itemsize == 4) + if (offsets->itemsize == 4) return offsets2parents_32bit(offsets, parents); - if (offsets.itemsize == 8) + if (offsets->itemsize == 8) return offsets2parents_64bit(offsets, parents); return 0; } -int counts2offsets_8bit(struct c_array counts, struct c_array offsets) { - ssize_t N = counts.strides[0] / counts.itemsize; - ((int8_t*)offsets.ptr)[0] = 0; - for (ssize_t i = 0; i < counts.size; i++) - ((int8_t*)offsets.ptr)[i + 1] = ((int8_t*)offsets.ptr)[i] + ((int8_t*)counts.ptr)[i * N]; +int counts2offsets_8bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int8_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int8_t*)offsets->ptr)[i + 1] = ((int8_t*)offsets->ptr)[i] + ((int8_t*)counts->ptr)[i * N]; return 1; } -int counts2offsets_16bit(struct c_array counts, struct c_array offsets) { - ssize_t N = counts.strides[0] / counts.itemsize; - ((int16_t*)offsets.ptr)[0] = 0; - for (ssize_t i = 0; i < counts.size; i++) - ((int16_t*)offsets.ptr)[i + 1] = ((int16_t*)offsets.ptr)[i] + ((int16_t*)counts.ptr)[i * N]; +int counts2offsets_16bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int16_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int16_t*)offsets->ptr)[i + 1] = ((int16_t*)offsets->ptr)[i] + ((int16_t*)counts->ptr)[i * N]; return 1; } -int counts2offsets_32bit(struct c_array counts, struct c_array offsets) { - ssize_t N = counts.strides[0] / counts.itemsize; - ((int32_t*)offsets.ptr)[0] = 0; - for (ssize_t i = 0; i < counts.size; i++) - ((int32_t*)offsets.ptr)[i + 1] = ((int32_t*)offsets.ptr)[i] + ((int32_t*)counts.ptr)[i * N]; +int counts2offsets_32bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int32_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int32_t*)offsets->ptr)[i + 1] = ((int32_t*)offsets->ptr)[i] + ((int32_t*)counts->ptr)[i * N]; return 1; } -int counts2offsets_64bit(struct c_array counts, struct c_array offsets) { - ssize_t N = counts.strides[0] / counts.itemsize; - ((int64_t*)offsets.ptr)[0] = 0; - for (ssize_t i = 0; i < counts.size; i++) - ((int64_t*)offsets.ptr)[i + 1] = ((int64_t*)offsets.ptr)[i] + ((int64_t*)counts.ptr)[i * N]; +int counts2offsets_64bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int64_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int64_t*)offsets->ptr)[i + 1] = ((int64_t*)offsets->ptr)[i] + ((int64_t*)counts->ptr)[i * N]; return 1; } -int counts2offsets_CPU(struct c_array counts, struct c_array offsets) { +int counts2offsets_CPU(struct c_array *counts, struct c_array *offsets) { /* PURPOSE: - converts counts to offsets PREREQUISITES: - counts is a non-negative, 1d int array - - offsets is a NEW array of size offsets.size + 1 + - offsets is a NEW array of size offsets->size + 1 - counts and offsets are of the same type - note: for >1d counts, flatten it first */ - if (counts.itemsize == 1) + if (counts->itemsize == 1) return counts2offsets_8bit(counts, offsets); - if (counts.itemsize == 2) + if (counts->itemsize == 2) return counts2offsets_16bit(counts, offsets); - if (counts.itemsize == 4) + if (counts->itemsize == 4) return counts2offsets_32bit(counts, offsets); - if (counts.itemsize == 8) + if (counts->itemsize == 8) return counts2offsets_64bit(counts, offsets); return 0; } -int startsstops2parents_8bit(struct c_array starts, struct c_array stops, struct c_array parents) { - for (ssize_t i = 0; i < parents.size; i++) - ((int8_t*)parents.ptr)[i] = -1; - ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; - for (ssize_t i = 0; i < starts.size; i++) - for (ssize_t j = (ssize_t)((int8_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int8_t*)stops.ptr)[i * N_sto]; j++) - ((int8_t*)parents.ptr)[j] = (int8_t)i; +int startsstops2parents_8bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { + for (ssize_t i = 0; i < parents->size; i++) + ((int8_t*)parents->ptr)[i] = -1; + ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; + for (ssize_t i = 0; i < starts->size; i++) + for (ssize_t j = (ssize_t)((int8_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int8_t*)stops->ptr)[i * N_sto]; j++) + ((int8_t*)parents->ptr)[j] = (int8_t)i; return 1; } -int startsstops2parents_16bit(struct c_array starts, struct c_array stops, struct c_array parents) { - for (ssize_t i = 0; i < parents.size; i++) - ((int16_t*)parents.ptr)[i] = -1; - ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; - for (ssize_t i = 0; i < starts.size; i++) - for (ssize_t j = (ssize_t)((int16_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int16_t*)stops.ptr)[i * N_sto]; j++) - ((int16_t*)parents.ptr)[j] = (int16_t)i; +int startsstops2parents_16bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { + for (ssize_t i = 0; i < parents->size; i++) + ((int16_t*)parents->ptr)[i] = -1; + ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; + for (ssize_t i = 0; i < starts->size; i++) + for (ssize_t j = (ssize_t)((int16_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int16_t*)stops->ptr)[i * N_sto]; j++) + ((int16_t*)parents->ptr)[j] = (int16_t)i; return 1; } -int startsstops2parents_32bit(struct c_array starts, struct c_array stops, struct c_array parents) { - for (ssize_t i = 0; i < parents.size; i++) - ((int32_t*)parents.ptr)[i] = -1; - ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; - for (ssize_t i = 0; i < starts.size; i++) - for (ssize_t j = (ssize_t)((int32_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int32_t*)stops.ptr)[i * N_sto]; j++) - ((int32_t*)parents.ptr)[j] = (int32_t)i; +int startsstops2parents_32bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { + for (ssize_t i = 0; i < parents->size; i++) + ((int32_t*)parents->ptr)[i] = -1; + ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; + for (ssize_t i = 0; i < starts->size; i++) + for (ssize_t j = (ssize_t)((int32_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int32_t*)stops->ptr)[i * N_sto]; j++) + ((int32_t*)parents->ptr)[j] = (int32_t)i; return 1; } -int startsstops2parents_64bit(struct c_array starts, struct c_array stops, struct c_array parents) { - for (ssize_t i = 0; i < parents.size; i++) - ((int64_t*)parents.ptr)[i] = -1; - ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; - for (ssize_t i = 0; i < starts.size; i++) - for (ssize_t j = (ssize_t)((int64_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int64_t*)stops.ptr)[i * N_sto]; j++) - ((int64_t*)parents.ptr)[j] = (int64_t)i; +int startsstops2parents_64bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { + for (ssize_t i = 0; i < parents->size; i++) + ((int64_t*)parents->ptr)[i] = -1; + ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; + for (ssize_t i = 0; i < starts->size; i++) + for (ssize_t j = (ssize_t)((int64_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int64_t*)stops->ptr)[i * N_sto]; j++) + ((int64_t*)parents->ptr)[j] = (int64_t)i; return 1; } -int startsstops2parents_CPU(struct c_array starts, struct c_array stops, struct c_array parents) { +int startsstops2parents_CPU(struct c_array *starts, struct c_array *stops, struct c_array *parents) { /* PURPOSE: - converts starts and stops to parents PREREQUISITES: - starts and stops are non-negative 1d int arrays - starts must be less than or equal to stops - starts must be of equal or shorter length than stops - - parents is a NEW array of length stops.maximum + - parents is a NEW array of length stops->maximum - starts, stops, and parents are of the same type - note: for >1d starts/stops, flatten them first */ - if (starts.itemsize == 1) + if (starts->itemsize == 1) return startsstops2parents_8bit(starts, stops, parents); - if (starts.itemsize == 2) + if (starts->itemsize == 2) return startsstops2parents_16bit(starts, stops, parents); - if (starts.itemsize == 4) + if (starts->itemsize == 4) return startsstops2parents_32bit(starts, stops, parents); - if (starts.itemsize == 8) + if (starts->itemsize == 8) return startsstops2parents_64bit(starts, stops, parents); return 0; } diff --git a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h index b370514f..a789a733 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h +++ b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h @@ -24,38 +24,38 @@ struct c_array py2c(py::array input) { } int makeIntNative_CPU(py::array input) { - if (!checkInt_CPU(py2c(input))) { + if (!checkInt_CPU(&py2c(input))) { throw std::invalid_argument("Argument must be an int array"); } - if (!makeNative_CPU(py2c(input))) { + if (!makeNative_CPU(&py2c(input))) { throw std::invalid_argument("Error in cpu_methods.h::makeNative_CPU"); } return 1; } int getMax_CPU(py::array input, std::int8_t* max) { - if (!getMax_8bit(py2c(input), 0, 0, max)) { + if (!getMax_8bit(&py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_8bit"); } return 1; } int getMax_CPU(py::array input, std::int16_t* max) { - if (!getMax_16bit(py2c(input), 0, 0, max)) { + if (!getMax_16bit(&py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_16bit"); } return 1; } int getMax_CPU(py::array input, std::int32_t* max) { - if (!getMax_32bit(py2c(input), 0, 0, max)) { + if (!getMax_32bit(&py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_32bit"); } return 1; } int getMax_CPU(py::array input, std::int64_t* max) { - if (!getMax_64bit(py2c(input), 0, 0, max)) { + if (!getMax_64bit(&py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_64bit"); } return 1; diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index b3996a8d..2104cef6 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -313,7 +313,7 @@ class JaggedArray : public AwkwardArray { ssize_t parents_length = (ssize_t)offsets_ptr[(offsets_info.size - 1) * N]; auto parents = py::array_t(parents_length); - if (!offsets2parents_CPU(py2c(offsets), py2c(parents))) { + if (!offsets2parents_CPU(&py2c(offsets), &py2c(parents))) { throw std::invalid_argument("Error in cpu_methods.h::offsets2parents_CPU"); } return parents; @@ -328,7 +328,7 @@ class JaggedArray : public AwkwardArray { makeIntNative_CPU(counts); counts = counts.cast>(); auto offsets = py::array_t(counts.request().size + 1); - if (!counts2offsets_CPU(py2c(counts), py2c(offsets))) { + if (!counts2offsets_CPU(&py2c(counts), &py2c(offsets))) { throw std::invalid_argument("Error in cpu_methods.h::counts2offsets_CPU"); } return offsets; @@ -349,7 +349,7 @@ class JaggedArray : public AwkwardArray { getMax_CPU(stops_, &max); auto parents = py::array_t((ssize_t)max); - if (!startsstops2parents_CPU(py2c(starts_), py2c(stops_), py2c(parents))) { + if (!startsstops2parents_CPU(&py2c(starts_), &py2c(stops_), &py2c(parents))) { throw std::invalid_argument("Error in cpu_methods.h::startsstops2parents_CPU"); } return parents; From 76e4a7106d426e65fca469ec8eb75e93c7fd8001 Mon Sep 17 00:00:00 2001 From: EscottC Date: Fri, 12 Jul 2019 14:38:46 -0700 Subject: [PATCH 21/26] Revert "fixing travis errors attempt #2" This reverts commit 5c66059fb2ffd5e7c2db5cafa4fbadde0a6ad27a. --- awkward-cpp/awkward/cpp/array/cpu_methods.h | 316 +++++++++---------- awkward-cpp/awkward/cpp/array/cpu_pybind11.h | 12 +- awkward-cpp/awkward/cpp/array/jagged.h | 6 +- 3 files changed, 167 insertions(+), 167 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/cpu_methods.h b/awkward-cpp/awkward/cpp/array/cpu_methods.h index 3606459e..f9c2d317 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_methods.h +++ b/awkward-cpp/awkward/cpp/array/cpu_methods.h @@ -35,62 +35,62 @@ int byteswap_64bit(int64_t *val) { return 1; } -int isNative_CPU(struct c_array *input) { +int isNative_CPU(struct c_array input) { // returns 1 if input is native, or 0 if non-isNative union { uint32_t i; char c[4]; } bint = { 0x01020304 }; - return ((bint.c[0] == 1 && input->format[0] != '<') - || (bint.c[0] != 1 && input->format[0] != '>')); + return ((bint.c[0] == 1 && input.format[0] != '<') + || (bint.c[0] != 1 && input.format[0] != '>')); } -int makeNative_16bit(struct c_array *input, ssize_t dim, ssize_t index) { +int makeNative_16bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_16bit(input, 0, 0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - byteswap_16bit(&((int16_t*)input->ptr)[index + i * N]); + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + byteswap_16bit(&((int16_t*)input.ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) makeNative_16bit(input, dim + 1, index + i * N); return 1; } -int makeNative_32bit(struct c_array *input, ssize_t dim, ssize_t index) { +int makeNative_32bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_32bit(input, 0, 0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - byteswap_32bit(&((int32_t*)input->ptr)[index + i * N]); + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + byteswap_32bit(&((int32_t*)input.ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) makeNative_32bit(input, dim + 1, index + i * N); return 1; } -int makeNative_64bit(struct c_array *input, ssize_t dim, ssize_t index) { +int makeNative_64bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_64bit(input, 0, 0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - byteswap_64bit(&((int64_t*)input->ptr)[index + i * N]); + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + byteswap_64bit(&((int64_t*)input.ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) makeNative_64bit(input, dim + 1, index + i * N); return 1; } -int makeNative_CPU(struct c_array *input) { +int makeNative_CPU(struct c_array input) { /* PURPOSE: - checks if input array is a native array - if it is non-native, it makes it native @@ -99,163 +99,163 @@ int makeNative_CPU(struct c_array *input) { */ if (isNative_CPU(input)) return 1; - if (input->itemsize == 1) + if (input.itemsize == 1) return 1; - if (input->itemsize == 2) + if (input.itemsize == 2) return makeNative_16bit(input, 0, 0); - if (input->itemsize == 4) + if (input.itemsize == 4) return makeNative_32bit(input, 0, 0); - if (input->itemsize == 8) + if (input.itemsize == 8) return makeNative_64bit(input, 0, 0); return 0; } -int checkInt_CPU(struct c_array *input) { +int checkInt_CPU(struct c_array input) { // returns 1 if it's an int array char intList[] = "qQlLhHbB"; ssize_t k = 0; - if (input->format[0] == '<' || input->format[0] == '>' || - input->format[0] == '=') + if (input.format[0] == '<' || input.format[0] == '>' || + input.format[0] == '=') k = 1; for (ssize_t i = 0; i < 8; i++) - if (intList[i] == input->format[k]) + if (intList[i] == input.format[k]) return 1; return 0; } -int getMax_8bit(struct c_array *input, ssize_t dim, ssize_t index, int8_t *max) { +int getMax_8bit(struct c_array input, ssize_t dim, ssize_t index, int8_t *max) { // to be initially called with getMax_8bit(input, 0, 0, *0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; if (dim == 0) { - if (input->shape[0] == 0) { + if (input.shape[0] == 0) { *max = 0; return 1; } - *max = ((int8_t*)input->ptr)[0]; + *max = ((int8_t*)input.ptr)[0]; } - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - if (((int8_t*)input->ptr)[index + i * N] > *max) - *max = ((int8_t*)input->ptr)[index + i * N]; + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int8_t*)input.ptr)[index + i * N] > *max) + *max = ((int8_t*)input.ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) getMax_8bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_16bit(struct c_array *input, ssize_t dim, ssize_t index, int16_t *max) { +int getMax_16bit(struct c_array input, ssize_t dim, ssize_t index, int16_t *max) { // to be initially called with getMax_16bit(input, 0, 0, *0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; if (dim == 0) { - if (input->shape[0] == 0) { + if (input.shape[0] == 0) { *max = 0; return 1; } - *max = ((int16_t*)input->ptr)[0]; + *max = ((int16_t*)input.ptr)[0]; } - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - if (((int16_t*)input->ptr)[index + i * N] > *max) - *max = ((int16_t*)input->ptr)[index + i * N]; + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int16_t*)input.ptr)[index + i * N] > *max) + *max = ((int16_t*)input.ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) getMax_16bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_32bit(struct c_array *input, ssize_t dim, ssize_t index, int32_t *max) { +int getMax_32bit(struct c_array input, ssize_t dim, ssize_t index, int32_t *max) { // to be initially called with getMax_32bit(input, 0, 0, *0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; if (dim == 0) { - if (input->shape[0] == 0) { + if (input.shape[0] == 0) { *max = 0; return 1; } - *max = ((int32_t*)input->ptr)[0]; + *max = ((int32_t*)input.ptr)[0]; } - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - if (((int32_t*)input->ptr)[index + i * N] > *max) - *max = ((int32_t*)input->ptr)[index + i * N]; + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int32_t*)input.ptr)[index + i * N] > *max) + *max = ((int32_t*)input.ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) getMax_32bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_64bit(struct c_array *input, ssize_t dim, ssize_t index, int64_t *max) { +int getMax_64bit(struct c_array input, ssize_t dim, ssize_t index, int64_t *max) { // to be initially called with getMax_64bit(input, 0, 0, *0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; if (dim == 0) { - if (input->shape[0] == 0) { + if (input.shape[0] == 0) { *max = 0; return 1; } - *max = ((int64_t*)input->ptr)[0]; + *max = ((int64_t*)input.ptr)[0]; } - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - if (((int64_t*)input->ptr)[index + i * N] > *max) - *max = ((int64_t*)input->ptr)[index + i * N]; + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int64_t*)input.ptr)[index + i * N] > *max) + *max = ((int64_t*)input.ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) getMax_64bit(input, dim + 1, index + i * N, max); return 1; } -int offsets2parents_8bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int8_t*)offsets->ptr)[i * N]) - ((int8_t*)parents->ptr)[j++] = (int8_t)k; +int offsets2parents_8bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int8_t*)offsets.ptr)[i * N]) + ((int8_t*)parents.ptr)[j++] = (int8_t)k; k++; } return 1; } -int offsets2parents_16bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int16_t*)offsets->ptr)[i * N]) - ((int16_t*)parents->ptr)[j++] = (int16_t)k; +int offsets2parents_16bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int16_t*)offsets.ptr)[i * N]) + ((int16_t*)parents.ptr)[j++] = (int16_t)k; k++; } return 1; } -int offsets2parents_32bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int32_t*)offsets->ptr)[i * N]) - ((int32_t*)parents->ptr)[j++] = (int32_t)k; +int offsets2parents_32bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int32_t*)offsets.ptr)[i * N]) + ((int32_t*)parents.ptr)[j++] = (int32_t)k; k++; } return 1; } -int offsets2parents_64bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int64_t*)offsets->ptr)[i * N]) - ((int64_t*)parents->ptr)[j++] = (int64_t)k; +int offsets2parents_64bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int64_t*)offsets.ptr)[i * N]) + ((int64_t*)parents.ptr)[j++] = (int64_t)k; k++; } return 1; } -int offsets2parents_CPU(struct c_array *offsets, struct c_array *parents) { +int offsets2parents_CPU(struct c_array offsets, struct c_array parents) { /* PURPOSE: - converts offsets to parents PREREQUISITES: @@ -263,127 +263,127 @@ int offsets2parents_CPU(struct c_array *offsets, struct c_array *parents) { - parents is a NEW array of size offsets[-1] - offsets and parents are of the same type */ - if (offsets->itemsize == 1) + if (offsets.itemsize == 1) return offsets2parents_8bit(offsets, parents); - if (offsets->itemsize == 2) + if (offsets.itemsize == 2) return offsets2parents_16bit(offsets, parents); - if (offsets->itemsize == 4) + if (offsets.itemsize == 4) return offsets2parents_32bit(offsets, parents); - if (offsets->itemsize == 8) + if (offsets.itemsize == 8) return offsets2parents_64bit(offsets, parents); return 0; } -int counts2offsets_8bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int8_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int8_t*)offsets->ptr)[i + 1] = ((int8_t*)offsets->ptr)[i] + ((int8_t*)counts->ptr)[i * N]; +int counts2offsets_8bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int8_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int8_t*)offsets.ptr)[i + 1] = ((int8_t*)offsets.ptr)[i] + ((int8_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_16bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int16_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int16_t*)offsets->ptr)[i + 1] = ((int16_t*)offsets->ptr)[i] + ((int16_t*)counts->ptr)[i * N]; +int counts2offsets_16bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int16_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int16_t*)offsets.ptr)[i + 1] = ((int16_t*)offsets.ptr)[i] + ((int16_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_32bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int32_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int32_t*)offsets->ptr)[i + 1] = ((int32_t*)offsets->ptr)[i] + ((int32_t*)counts->ptr)[i * N]; +int counts2offsets_32bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int32_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int32_t*)offsets.ptr)[i + 1] = ((int32_t*)offsets.ptr)[i] + ((int32_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_64bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int64_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int64_t*)offsets->ptr)[i + 1] = ((int64_t*)offsets->ptr)[i] + ((int64_t*)counts->ptr)[i * N]; +int counts2offsets_64bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int64_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int64_t*)offsets.ptr)[i + 1] = ((int64_t*)offsets.ptr)[i] + ((int64_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_CPU(struct c_array *counts, struct c_array *offsets) { +int counts2offsets_CPU(struct c_array counts, struct c_array offsets) { /* PURPOSE: - converts counts to offsets PREREQUISITES: - counts is a non-negative, 1d int array - - offsets is a NEW array of size offsets->size + 1 + - offsets is a NEW array of size offsets.size + 1 - counts and offsets are of the same type - note: for >1d counts, flatten it first */ - if (counts->itemsize == 1) + if (counts.itemsize == 1) return counts2offsets_8bit(counts, offsets); - if (counts->itemsize == 2) + if (counts.itemsize == 2) return counts2offsets_16bit(counts, offsets); - if (counts->itemsize == 4) + if (counts.itemsize == 4) return counts2offsets_32bit(counts, offsets); - if (counts->itemsize == 8) + if (counts.itemsize == 8) return counts2offsets_64bit(counts, offsets); return 0; } -int startsstops2parents_8bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { - for (ssize_t i = 0; i < parents->size; i++) - ((int8_t*)parents->ptr)[i] = -1; - ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; - for (ssize_t i = 0; i < starts->size; i++) - for (ssize_t j = (ssize_t)((int8_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int8_t*)stops->ptr)[i * N_sto]; j++) - ((int8_t*)parents->ptr)[j] = (int8_t)i; +int startsstops2parents_8bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int8_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int8_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int8_t*)stops.ptr)[i * N_sto]; j++) + ((int8_t*)parents.ptr)[j] = (int8_t)i; return 1; } -int startsstops2parents_16bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { - for (ssize_t i = 0; i < parents->size; i++) - ((int16_t*)parents->ptr)[i] = -1; - ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; - for (ssize_t i = 0; i < starts->size; i++) - for (ssize_t j = (ssize_t)((int16_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int16_t*)stops->ptr)[i * N_sto]; j++) - ((int16_t*)parents->ptr)[j] = (int16_t)i; +int startsstops2parents_16bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int16_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int16_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int16_t*)stops.ptr)[i * N_sto]; j++) + ((int16_t*)parents.ptr)[j] = (int16_t)i; return 1; } -int startsstops2parents_32bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { - for (ssize_t i = 0; i < parents->size; i++) - ((int32_t*)parents->ptr)[i] = -1; - ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; - for (ssize_t i = 0; i < starts->size; i++) - for (ssize_t j = (ssize_t)((int32_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int32_t*)stops->ptr)[i * N_sto]; j++) - ((int32_t*)parents->ptr)[j] = (int32_t)i; +int startsstops2parents_32bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int32_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int32_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int32_t*)stops.ptr)[i * N_sto]; j++) + ((int32_t*)parents.ptr)[j] = (int32_t)i; return 1; } -int startsstops2parents_64bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { - for (ssize_t i = 0; i < parents->size; i++) - ((int64_t*)parents->ptr)[i] = -1; - ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; - for (ssize_t i = 0; i < starts->size; i++) - for (ssize_t j = (ssize_t)((int64_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int64_t*)stops->ptr)[i * N_sto]; j++) - ((int64_t*)parents->ptr)[j] = (int64_t)i; +int startsstops2parents_64bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int64_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int64_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int64_t*)stops.ptr)[i * N_sto]; j++) + ((int64_t*)parents.ptr)[j] = (int64_t)i; return 1; } -int startsstops2parents_CPU(struct c_array *starts, struct c_array *stops, struct c_array *parents) { +int startsstops2parents_CPU(struct c_array starts, struct c_array stops, struct c_array parents) { /* PURPOSE: - converts starts and stops to parents PREREQUISITES: - starts and stops are non-negative 1d int arrays - starts must be less than or equal to stops - starts must be of equal or shorter length than stops - - parents is a NEW array of length stops->maximum + - parents is a NEW array of length stops.maximum - starts, stops, and parents are of the same type - note: for >1d starts/stops, flatten them first */ - if (starts->itemsize == 1) + if (starts.itemsize == 1) return startsstops2parents_8bit(starts, stops, parents); - if (starts->itemsize == 2) + if (starts.itemsize == 2) return startsstops2parents_16bit(starts, stops, parents); - if (starts->itemsize == 4) + if (starts.itemsize == 4) return startsstops2parents_32bit(starts, stops, parents); - if (starts->itemsize == 8) + if (starts.itemsize == 8) return startsstops2parents_64bit(starts, stops, parents); return 0; } diff --git a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h index a789a733..b370514f 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h +++ b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h @@ -24,38 +24,38 @@ struct c_array py2c(py::array input) { } int makeIntNative_CPU(py::array input) { - if (!checkInt_CPU(&py2c(input))) { + if (!checkInt_CPU(py2c(input))) { throw std::invalid_argument("Argument must be an int array"); } - if (!makeNative_CPU(&py2c(input))) { + if (!makeNative_CPU(py2c(input))) { throw std::invalid_argument("Error in cpu_methods.h::makeNative_CPU"); } return 1; } int getMax_CPU(py::array input, std::int8_t* max) { - if (!getMax_8bit(&py2c(input), 0, 0, max)) { + if (!getMax_8bit(py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_8bit"); } return 1; } int getMax_CPU(py::array input, std::int16_t* max) { - if (!getMax_16bit(&py2c(input), 0, 0, max)) { + if (!getMax_16bit(py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_16bit"); } return 1; } int getMax_CPU(py::array input, std::int32_t* max) { - if (!getMax_32bit(&py2c(input), 0, 0, max)) { + if (!getMax_32bit(py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_32bit"); } return 1; } int getMax_CPU(py::array input, std::int64_t* max) { - if (!getMax_64bit(&py2c(input), 0, 0, max)) { + if (!getMax_64bit(py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_64bit"); } return 1; diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index 2104cef6..b3996a8d 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -313,7 +313,7 @@ class JaggedArray : public AwkwardArray { ssize_t parents_length = (ssize_t)offsets_ptr[(offsets_info.size - 1) * N]; auto parents = py::array_t(parents_length); - if (!offsets2parents_CPU(&py2c(offsets), &py2c(parents))) { + if (!offsets2parents_CPU(py2c(offsets), py2c(parents))) { throw std::invalid_argument("Error in cpu_methods.h::offsets2parents_CPU"); } return parents; @@ -328,7 +328,7 @@ class JaggedArray : public AwkwardArray { makeIntNative_CPU(counts); counts = counts.cast>(); auto offsets = py::array_t(counts.request().size + 1); - if (!counts2offsets_CPU(&py2c(counts), &py2c(offsets))) { + if (!counts2offsets_CPU(py2c(counts), py2c(offsets))) { throw std::invalid_argument("Error in cpu_methods.h::counts2offsets_CPU"); } return offsets; @@ -349,7 +349,7 @@ class JaggedArray : public AwkwardArray { getMax_CPU(stops_, &max); auto parents = py::array_t((ssize_t)max); - if (!startsstops2parents_CPU(&py2c(starts_), &py2c(stops_), &py2c(parents))) { + if (!startsstops2parents_CPU(py2c(starts_), py2c(stops_), py2c(parents))) { throw std::invalid_argument("Error in cpu_methods.h::startsstops2parents_CPU"); } return parents; From f4d991d0b3df30b9c2d31eebc13440d44e126d14 Mon Sep 17 00:00:00 2001 From: EscottC Date: Tue, 16 Jul 2019 14:36:49 -0700 Subject: [PATCH 22/26] memory issues in cpu_pybind11.h --- awkward-cpp/awkward/cpp/array/cpu_methods.h | 316 +++++++++---------- awkward-cpp/awkward/cpp/array/cpu_pybind11.h | 45 +-- awkward-cpp/awkward/cpp/array/jagged.h | 24 +- 3 files changed, 206 insertions(+), 179 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/cpu_methods.h b/awkward-cpp/awkward/cpp/array/cpu_methods.h index f9c2d317..3606459e 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_methods.h +++ b/awkward-cpp/awkward/cpp/array/cpu_methods.h @@ -35,62 +35,62 @@ int byteswap_64bit(int64_t *val) { return 1; } -int isNative_CPU(struct c_array input) { +int isNative_CPU(struct c_array *input) { // returns 1 if input is native, or 0 if non-isNative union { uint32_t i; char c[4]; } bint = { 0x01020304 }; - return ((bint.c[0] == 1 && input.format[0] != '<') - || (bint.c[0] != 1 && input.format[0] != '>')); + return ((bint.c[0] == 1 && input->format[0] != '<') + || (bint.c[0] != 1 && input->format[0] != '>')); } -int makeNative_16bit(struct c_array input, ssize_t dim, ssize_t index) { +int makeNative_16bit(struct c_array *input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_16bit(input, 0, 0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - byteswap_16bit(&((int16_t*)input.ptr)[index + i * N]); + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + byteswap_16bit(&((int16_t*)input->ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) makeNative_16bit(input, dim + 1, index + i * N); return 1; } -int makeNative_32bit(struct c_array input, ssize_t dim, ssize_t index) { +int makeNative_32bit(struct c_array *input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_32bit(input, 0, 0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - byteswap_32bit(&((int32_t*)input.ptr)[index + i * N]); + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + byteswap_32bit(&((int32_t*)input->ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) makeNative_32bit(input, dim + 1, index + i * N); return 1; } -int makeNative_64bit(struct c_array input, ssize_t dim, ssize_t index) { +int makeNative_64bit(struct c_array *input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_64bit(input, 0, 0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - byteswap_64bit(&((int64_t*)input.ptr)[index + i * N]); + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + byteswap_64bit(&((int64_t*)input->ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) makeNative_64bit(input, dim + 1, index + i * N); return 1; } -int makeNative_CPU(struct c_array input) { +int makeNative_CPU(struct c_array *input) { /* PURPOSE: - checks if input array is a native array - if it is non-native, it makes it native @@ -99,163 +99,163 @@ int makeNative_CPU(struct c_array input) { */ if (isNative_CPU(input)) return 1; - if (input.itemsize == 1) + if (input->itemsize == 1) return 1; - if (input.itemsize == 2) + if (input->itemsize == 2) return makeNative_16bit(input, 0, 0); - if (input.itemsize == 4) + if (input->itemsize == 4) return makeNative_32bit(input, 0, 0); - if (input.itemsize == 8) + if (input->itemsize == 8) return makeNative_64bit(input, 0, 0); return 0; } -int checkInt_CPU(struct c_array input) { +int checkInt_CPU(struct c_array *input) { // returns 1 if it's an int array char intList[] = "qQlLhHbB"; ssize_t k = 0; - if (input.format[0] == '<' || input.format[0] == '>' || - input.format[0] == '=') + if (input->format[0] == '<' || input->format[0] == '>' || + input->format[0] == '=') k = 1; for (ssize_t i = 0; i < 8; i++) - if (intList[i] == input.format[k]) + if (intList[i] == input->format[k]) return 1; return 0; } -int getMax_8bit(struct c_array input, ssize_t dim, ssize_t index, int8_t *max) { +int getMax_8bit(struct c_array *input, ssize_t dim, ssize_t index, int8_t *max) { // to be initially called with getMax_8bit(input, 0, 0, *0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; if (dim == 0) { - if (input.shape[0] == 0) { + if (input->shape[0] == 0) { *max = 0; return 1; } - *max = ((int8_t*)input.ptr)[0]; + *max = ((int8_t*)input->ptr)[0]; } - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - if (((int8_t*)input.ptr)[index + i * N] > *max) - *max = ((int8_t*)input.ptr)[index + i * N]; + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + if (((int8_t*)input->ptr)[index + i * N] > *max) + *max = ((int8_t*)input->ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) getMax_8bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_16bit(struct c_array input, ssize_t dim, ssize_t index, int16_t *max) { +int getMax_16bit(struct c_array *input, ssize_t dim, ssize_t index, int16_t *max) { // to be initially called with getMax_16bit(input, 0, 0, *0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; if (dim == 0) { - if (input.shape[0] == 0) { + if (input->shape[0] == 0) { *max = 0; return 1; } - *max = ((int16_t*)input.ptr)[0]; + *max = ((int16_t*)input->ptr)[0]; } - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - if (((int16_t*)input.ptr)[index + i * N] > *max) - *max = ((int16_t*)input.ptr)[index + i * N]; + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + if (((int16_t*)input->ptr)[index + i * N] > *max) + *max = ((int16_t*)input->ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) getMax_16bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_32bit(struct c_array input, ssize_t dim, ssize_t index, int32_t *max) { +int getMax_32bit(struct c_array *input, ssize_t dim, ssize_t index, int32_t *max) { // to be initially called with getMax_32bit(input, 0, 0, *0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; if (dim == 0) { - if (input.shape[0] == 0) { + if (input->shape[0] == 0) { *max = 0; return 1; } - *max = ((int32_t*)input.ptr)[0]; + *max = ((int32_t*)input->ptr)[0]; } - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - if (((int32_t*)input.ptr)[index + i * N] > *max) - *max = ((int32_t*)input.ptr)[index + i * N]; + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + if (((int32_t*)input->ptr)[index + i * N] > *max) + *max = ((int32_t*)input->ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) getMax_32bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_64bit(struct c_array input, ssize_t dim, ssize_t index, int64_t *max) { +int getMax_64bit(struct c_array *input, ssize_t dim, ssize_t index, int64_t *max) { // to be initially called with getMax_64bit(input, 0, 0, *0) - if (dim > input.ndim - 1) + if (dim > input->ndim - 1) return 0; if (dim == 0) { - if (input.shape[0] == 0) { + if (input->shape[0] == 0) { *max = 0; return 1; } - *max = ((int64_t*)input.ptr)[0]; + *max = ((int64_t*)input->ptr)[0]; } - ssize_t N = input.strides[dim] / input.itemsize; - if (dim == input.ndim - 1) { - for (ssize_t i = 0; i < input.shape[dim]; i++) - if (((int64_t*)input.ptr)[index + i * N] > *max) - *max = ((int64_t*)input.ptr)[index + i * N]; + ssize_t N = input->strides[dim] / input->itemsize; + if (dim == input->ndim - 1) { + for (ssize_t i = 0; i < input->shape[dim]; i++) + if (((int64_t*)input->ptr)[index + i * N] > *max) + *max = ((int64_t*)input->ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input.shape[dim]; i++) + for (ssize_t i = 0; i < input->shape[dim]; i++) getMax_64bit(input, dim + 1, index + i * N, max); return 1; } -int offsets2parents_8bit(struct c_array offsets, struct c_array parents) { - ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets.size; i++) { - while (j < (ssize_t)((int8_t*)offsets.ptr)[i * N]) - ((int8_t*)parents.ptr)[j++] = (int8_t)k; +int offsets2parents_8bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int8_t*)offsets->ptr)[i * N]) + ((int8_t*)parents->ptr)[j++] = (int8_t)k; k++; } return 1; } -int offsets2parents_16bit(struct c_array offsets, struct c_array parents) { - ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets.size; i++) { - while (j < (ssize_t)((int16_t*)offsets.ptr)[i * N]) - ((int16_t*)parents.ptr)[j++] = (int16_t)k; +int offsets2parents_16bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int16_t*)offsets->ptr)[i * N]) + ((int16_t*)parents->ptr)[j++] = (int16_t)k; k++; } return 1; } -int offsets2parents_32bit(struct c_array offsets, struct c_array parents) { - ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets.size; i++) { - while (j < (ssize_t)((int32_t*)offsets.ptr)[i * N]) - ((int32_t*)parents.ptr)[j++] = (int32_t)k; +int offsets2parents_32bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int32_t*)offsets->ptr)[i * N]) + ((int32_t*)parents->ptr)[j++] = (int32_t)k; k++; } return 1; } -int offsets2parents_64bit(struct c_array offsets, struct c_array parents) { - ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets.size; i++) { - while (j < (ssize_t)((int64_t*)offsets.ptr)[i * N]) - ((int64_t*)parents.ptr)[j++] = (int64_t)k; +int offsets2parents_64bit(struct c_array *offsets, struct c_array *parents) { + ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets->size; i++) { + while (j < (ssize_t)((int64_t*)offsets->ptr)[i * N]) + ((int64_t*)parents->ptr)[j++] = (int64_t)k; k++; } return 1; } -int offsets2parents_CPU(struct c_array offsets, struct c_array parents) { +int offsets2parents_CPU(struct c_array *offsets, struct c_array *parents) { /* PURPOSE: - converts offsets to parents PREREQUISITES: @@ -263,127 +263,127 @@ int offsets2parents_CPU(struct c_array offsets, struct c_array parents) { - parents is a NEW array of size offsets[-1] - offsets and parents are of the same type */ - if (offsets.itemsize == 1) + if (offsets->itemsize == 1) return offsets2parents_8bit(offsets, parents); - if (offsets.itemsize == 2) + if (offsets->itemsize == 2) return offsets2parents_16bit(offsets, parents); - if (offsets.itemsize == 4) + if (offsets->itemsize == 4) return offsets2parents_32bit(offsets, parents); - if (offsets.itemsize == 8) + if (offsets->itemsize == 8) return offsets2parents_64bit(offsets, parents); return 0; } -int counts2offsets_8bit(struct c_array counts, struct c_array offsets) { - ssize_t N = counts.strides[0] / counts.itemsize; - ((int8_t*)offsets.ptr)[0] = 0; - for (ssize_t i = 0; i < counts.size; i++) - ((int8_t*)offsets.ptr)[i + 1] = ((int8_t*)offsets.ptr)[i] + ((int8_t*)counts.ptr)[i * N]; +int counts2offsets_8bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int8_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int8_t*)offsets->ptr)[i + 1] = ((int8_t*)offsets->ptr)[i] + ((int8_t*)counts->ptr)[i * N]; return 1; } -int counts2offsets_16bit(struct c_array counts, struct c_array offsets) { - ssize_t N = counts.strides[0] / counts.itemsize; - ((int16_t*)offsets.ptr)[0] = 0; - for (ssize_t i = 0; i < counts.size; i++) - ((int16_t*)offsets.ptr)[i + 1] = ((int16_t*)offsets.ptr)[i] + ((int16_t*)counts.ptr)[i * N]; +int counts2offsets_16bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int16_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int16_t*)offsets->ptr)[i + 1] = ((int16_t*)offsets->ptr)[i] + ((int16_t*)counts->ptr)[i * N]; return 1; } -int counts2offsets_32bit(struct c_array counts, struct c_array offsets) { - ssize_t N = counts.strides[0] / counts.itemsize; - ((int32_t*)offsets.ptr)[0] = 0; - for (ssize_t i = 0; i < counts.size; i++) - ((int32_t*)offsets.ptr)[i + 1] = ((int32_t*)offsets.ptr)[i] + ((int32_t*)counts.ptr)[i * N]; +int counts2offsets_32bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int32_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int32_t*)offsets->ptr)[i + 1] = ((int32_t*)offsets->ptr)[i] + ((int32_t*)counts->ptr)[i * N]; return 1; } -int counts2offsets_64bit(struct c_array counts, struct c_array offsets) { - ssize_t N = counts.strides[0] / counts.itemsize; - ((int64_t*)offsets.ptr)[0] = 0; - for (ssize_t i = 0; i < counts.size; i++) - ((int64_t*)offsets.ptr)[i + 1] = ((int64_t*)offsets.ptr)[i] + ((int64_t*)counts.ptr)[i * N]; +int counts2offsets_64bit(struct c_array *counts, struct c_array *offsets) { + ssize_t N = counts->strides[0] / counts->itemsize; + ((int64_t*)offsets->ptr)[0] = 0; + for (ssize_t i = 0; i < counts->size; i++) + ((int64_t*)offsets->ptr)[i + 1] = ((int64_t*)offsets->ptr)[i] + ((int64_t*)counts->ptr)[i * N]; return 1; } -int counts2offsets_CPU(struct c_array counts, struct c_array offsets) { +int counts2offsets_CPU(struct c_array *counts, struct c_array *offsets) { /* PURPOSE: - converts counts to offsets PREREQUISITES: - counts is a non-negative, 1d int array - - offsets is a NEW array of size offsets.size + 1 + - offsets is a NEW array of size offsets->size + 1 - counts and offsets are of the same type - note: for >1d counts, flatten it first */ - if (counts.itemsize == 1) + if (counts->itemsize == 1) return counts2offsets_8bit(counts, offsets); - if (counts.itemsize == 2) + if (counts->itemsize == 2) return counts2offsets_16bit(counts, offsets); - if (counts.itemsize == 4) + if (counts->itemsize == 4) return counts2offsets_32bit(counts, offsets); - if (counts.itemsize == 8) + if (counts->itemsize == 8) return counts2offsets_64bit(counts, offsets); return 0; } -int startsstops2parents_8bit(struct c_array starts, struct c_array stops, struct c_array parents) { - for (ssize_t i = 0; i < parents.size; i++) - ((int8_t*)parents.ptr)[i] = -1; - ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; - for (ssize_t i = 0; i < starts.size; i++) - for (ssize_t j = (ssize_t)((int8_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int8_t*)stops.ptr)[i * N_sto]; j++) - ((int8_t*)parents.ptr)[j] = (int8_t)i; +int startsstops2parents_8bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { + for (ssize_t i = 0; i < parents->size; i++) + ((int8_t*)parents->ptr)[i] = -1; + ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; + for (ssize_t i = 0; i < starts->size; i++) + for (ssize_t j = (ssize_t)((int8_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int8_t*)stops->ptr)[i * N_sto]; j++) + ((int8_t*)parents->ptr)[j] = (int8_t)i; return 1; } -int startsstops2parents_16bit(struct c_array starts, struct c_array stops, struct c_array parents) { - for (ssize_t i = 0; i < parents.size; i++) - ((int16_t*)parents.ptr)[i] = -1; - ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; - for (ssize_t i = 0; i < starts.size; i++) - for (ssize_t j = (ssize_t)((int16_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int16_t*)stops.ptr)[i * N_sto]; j++) - ((int16_t*)parents.ptr)[j] = (int16_t)i; +int startsstops2parents_16bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { + for (ssize_t i = 0; i < parents->size; i++) + ((int16_t*)parents->ptr)[i] = -1; + ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; + for (ssize_t i = 0; i < starts->size; i++) + for (ssize_t j = (ssize_t)((int16_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int16_t*)stops->ptr)[i * N_sto]; j++) + ((int16_t*)parents->ptr)[j] = (int16_t)i; return 1; } -int startsstops2parents_32bit(struct c_array starts, struct c_array stops, struct c_array parents) { - for (ssize_t i = 0; i < parents.size; i++) - ((int32_t*)parents.ptr)[i] = -1; - ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; - for (ssize_t i = 0; i < starts.size; i++) - for (ssize_t j = (ssize_t)((int32_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int32_t*)stops.ptr)[i * N_sto]; j++) - ((int32_t*)parents.ptr)[j] = (int32_t)i; +int startsstops2parents_32bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { + for (ssize_t i = 0; i < parents->size; i++) + ((int32_t*)parents->ptr)[i] = -1; + ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; + for (ssize_t i = 0; i < starts->size; i++) + for (ssize_t j = (ssize_t)((int32_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int32_t*)stops->ptr)[i * N_sto]; j++) + ((int32_t*)parents->ptr)[j] = (int32_t)i; return 1; } -int startsstops2parents_64bit(struct c_array starts, struct c_array stops, struct c_array parents) { - for (ssize_t i = 0; i < parents.size; i++) - ((int64_t*)parents.ptr)[i] = -1; - ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; - for (ssize_t i = 0; i < starts.size; i++) - for (ssize_t j = (ssize_t)((int64_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int64_t*)stops.ptr)[i * N_sto]; j++) - ((int64_t*)parents.ptr)[j] = (int64_t)i; +int startsstops2parents_64bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { + for (ssize_t i = 0; i < parents->size; i++) + ((int64_t*)parents->ptr)[i] = -1; + ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; + for (ssize_t i = 0; i < starts->size; i++) + for (ssize_t j = (ssize_t)((int64_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int64_t*)stops->ptr)[i * N_sto]; j++) + ((int64_t*)parents->ptr)[j] = (int64_t)i; return 1; } -int startsstops2parents_CPU(struct c_array starts, struct c_array stops, struct c_array parents) { +int startsstops2parents_CPU(struct c_array *starts, struct c_array *stops, struct c_array *parents) { /* PURPOSE: - converts starts and stops to parents PREREQUISITES: - starts and stops are non-negative 1d int arrays - starts must be less than or equal to stops - starts must be of equal or shorter length than stops - - parents is a NEW array of length stops.maximum + - parents is a NEW array of length stops->maximum - starts, stops, and parents are of the same type - note: for >1d starts/stops, flatten them first */ - if (starts.itemsize == 1) + if (starts->itemsize == 1) return startsstops2parents_8bit(starts, stops, parents); - if (starts.itemsize == 2) + if (starts->itemsize == 2) return startsstops2parents_16bit(starts, stops, parents); - if (starts.itemsize == 4) + if (starts->itemsize == 4) return startsstops2parents_32bit(starts, stops, parents); - if (starts.itemsize == 8) + if (starts->itemsize == 8) return startsstops2parents_64bit(starts, stops, parents); return 0; } diff --git a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h index b370514f..221c2243 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h +++ b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h @@ -8,54 +8,63 @@ namespace py = pybind11; -struct c_array py2c(py::array input) { +int py2c(py::array py, struct c_array *c) { + py::buffer_info info = py.request(); char format[8]; - strcpy(format, input.request().format.c_str()); - struct c_array out = { - input.request().ptr, - input.request().itemsize, - input.request().size, - format, - input.request().ndim, - &input.request().shape[0], - &input.request().strides[0] - }; - return out; + strcpy(format, info.format.c_str()); + c->ptr = info.ptr; + c->itemsize = info.itemsize; + c->size = info.size; + c->format = format; + c->ndim = info.ndim; + c->shape = &info.shape[0]; + c->strides = &info.strides[0]; + return 1; } int makeIntNative_CPU(py::array input) { - if (!checkInt_CPU(py2c(input))) { + struct c_array array_struct; + py2c(input, &array_struct); + if (!checkInt_CPU(&array_struct)) { throw std::invalid_argument("Argument must be an int array"); } - if (!makeNative_CPU(py2c(input))) { + if (!makeNative_CPU(&array_struct)) { throw std::invalid_argument("Error in cpu_methods.h::makeNative_CPU"); } return 1; } int getMax_CPU(py::array input, std::int8_t* max) { - if (!getMax_8bit(py2c(input), 0, 0, max)) { + struct c_array array_struct; + py2c(input, &array_struct); + if (!getMax_8bit(&array_struct, 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_8bit"); } return 1; } int getMax_CPU(py::array input, std::int16_t* max) { - if (!getMax_16bit(py2c(input), 0, 0, max)) { + struct c_array array_struct; + py2c(input, &array_struct); + if (!getMax_16bit(&array_struct, 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_16bit"); } return 1; } int getMax_CPU(py::array input, std::int32_t* max) { - if (!getMax_32bit(py2c(input), 0, 0, max)) { + struct c_array array_struct; + py2c(input, &array_struct); + if (!getMax_32bit(&array_struct, 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_32bit"); } return 1; } int getMax_CPU(py::array input, std::int64_t* max) { - if (!getMax_64bit(py2c(input), 0, 0, max)) { + struct c_array array_struct; + py2c(input, &array_struct); + if (!getMax_64bit(&array_struct, 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_64bit"); } return 1; diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index b3996a8d..ae76cc1b 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -313,7 +313,12 @@ class JaggedArray : public AwkwardArray { ssize_t parents_length = (ssize_t)offsets_ptr[(offsets_info.size - 1) * N]; auto parents = py::array_t(parents_length); - if (!offsets2parents_CPU(py2c(offsets), py2c(parents))) { + struct c_array offsets_struct; + py2c(offsets, &offsets_struct); + struct c_array parents_struct; + py2c(parents, &parents_struct); + + if (!offsets2parents_CPU(&offsets_struct, &parents_struct)) { throw std::invalid_argument("Error in cpu_methods.h::offsets2parents_CPU"); } return parents; @@ -328,7 +333,13 @@ class JaggedArray : public AwkwardArray { makeIntNative_CPU(counts); counts = counts.cast>(); auto offsets = py::array_t(counts.request().size + 1); - if (!counts2offsets_CPU(py2c(counts), py2c(offsets))) { + + struct c_array counts_struct; + py2c(counts, &counts_struct); + struct c_array offsets_struct; + py2c(offsets, &offsets_struct); + + if (!counts2offsets_CPU(&counts_struct, &offsets_struct)) { throw std::invalid_argument("Error in cpu_methods.h::counts2offsets_CPU"); } return offsets; @@ -349,7 +360,14 @@ class JaggedArray : public AwkwardArray { getMax_CPU(stops_, &max); auto parents = py::array_t((ssize_t)max); - if (!startsstops2parents_CPU(py2c(starts_), py2c(stops_), py2c(parents))) { + struct c_array starts_struct; + py2c(starts_, &starts_struct); + struct c_array stops_struct; + py2c(stops_, &stops_struct); + struct c_array parents_struct; + py2c(parents, &parents_struct); + + if (!startsstops2parents_CPU(&starts_struct, &stops_struct, &parents_struct)) { throw std::invalid_argument("Error in cpu_methods.h::startsstops2parents_CPU"); } return parents; From f9bd610c5ed73b03527adfa933047d5a24b5fd12 Mon Sep 17 00:00:00 2001 From: EscottC Date: Tue, 16 Jul 2019 14:37:03 -0700 Subject: [PATCH 23/26] Revert "memory issues in cpu_pybind11.h" This reverts commit f4d991d0b3df30b9c2d31eebc13440d44e126d14. --- awkward-cpp/awkward/cpp/array/cpu_methods.h | 316 +++++++++---------- awkward-cpp/awkward/cpp/array/cpu_pybind11.h | 45 ++- awkward-cpp/awkward/cpp/array/jagged.h | 24 +- 3 files changed, 179 insertions(+), 206 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/cpu_methods.h b/awkward-cpp/awkward/cpp/array/cpu_methods.h index 3606459e..f9c2d317 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_methods.h +++ b/awkward-cpp/awkward/cpp/array/cpu_methods.h @@ -35,62 +35,62 @@ int byteswap_64bit(int64_t *val) { return 1; } -int isNative_CPU(struct c_array *input) { +int isNative_CPU(struct c_array input) { // returns 1 if input is native, or 0 if non-isNative union { uint32_t i; char c[4]; } bint = { 0x01020304 }; - return ((bint.c[0] == 1 && input->format[0] != '<') - || (bint.c[0] != 1 && input->format[0] != '>')); + return ((bint.c[0] == 1 && input.format[0] != '<') + || (bint.c[0] != 1 && input.format[0] != '>')); } -int makeNative_16bit(struct c_array *input, ssize_t dim, ssize_t index) { +int makeNative_16bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_16bit(input, 0, 0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - byteswap_16bit(&((int16_t*)input->ptr)[index + i * N]); + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + byteswap_16bit(&((int16_t*)input.ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) makeNative_16bit(input, dim + 1, index + i * N); return 1; } -int makeNative_32bit(struct c_array *input, ssize_t dim, ssize_t index) { +int makeNative_32bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_32bit(input, 0, 0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - byteswap_32bit(&((int32_t*)input->ptr)[index + i * N]); + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + byteswap_32bit(&((int32_t*)input.ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) makeNative_32bit(input, dim + 1, index + i * N); return 1; } -int makeNative_64bit(struct c_array *input, ssize_t dim, ssize_t index) { +int makeNative_64bit(struct c_array input, ssize_t dim, ssize_t index) { // to be initially called with makeNative_64bit(input, 0, 0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - byteswap_64bit(&((int64_t*)input->ptr)[index + i * N]); + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + byteswap_64bit(&((int64_t*)input.ptr)[index + i * N]); return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) makeNative_64bit(input, dim + 1, index + i * N); return 1; } -int makeNative_CPU(struct c_array *input) { +int makeNative_CPU(struct c_array input) { /* PURPOSE: - checks if input array is a native array - if it is non-native, it makes it native @@ -99,163 +99,163 @@ int makeNative_CPU(struct c_array *input) { */ if (isNative_CPU(input)) return 1; - if (input->itemsize == 1) + if (input.itemsize == 1) return 1; - if (input->itemsize == 2) + if (input.itemsize == 2) return makeNative_16bit(input, 0, 0); - if (input->itemsize == 4) + if (input.itemsize == 4) return makeNative_32bit(input, 0, 0); - if (input->itemsize == 8) + if (input.itemsize == 8) return makeNative_64bit(input, 0, 0); return 0; } -int checkInt_CPU(struct c_array *input) { +int checkInt_CPU(struct c_array input) { // returns 1 if it's an int array char intList[] = "qQlLhHbB"; ssize_t k = 0; - if (input->format[0] == '<' || input->format[0] == '>' || - input->format[0] == '=') + if (input.format[0] == '<' || input.format[0] == '>' || + input.format[0] == '=') k = 1; for (ssize_t i = 0; i < 8; i++) - if (intList[i] == input->format[k]) + if (intList[i] == input.format[k]) return 1; return 0; } -int getMax_8bit(struct c_array *input, ssize_t dim, ssize_t index, int8_t *max) { +int getMax_8bit(struct c_array input, ssize_t dim, ssize_t index, int8_t *max) { // to be initially called with getMax_8bit(input, 0, 0, *0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; if (dim == 0) { - if (input->shape[0] == 0) { + if (input.shape[0] == 0) { *max = 0; return 1; } - *max = ((int8_t*)input->ptr)[0]; + *max = ((int8_t*)input.ptr)[0]; } - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - if (((int8_t*)input->ptr)[index + i * N] > *max) - *max = ((int8_t*)input->ptr)[index + i * N]; + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int8_t*)input.ptr)[index + i * N] > *max) + *max = ((int8_t*)input.ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) getMax_8bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_16bit(struct c_array *input, ssize_t dim, ssize_t index, int16_t *max) { +int getMax_16bit(struct c_array input, ssize_t dim, ssize_t index, int16_t *max) { // to be initially called with getMax_16bit(input, 0, 0, *0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; if (dim == 0) { - if (input->shape[0] == 0) { + if (input.shape[0] == 0) { *max = 0; return 1; } - *max = ((int16_t*)input->ptr)[0]; + *max = ((int16_t*)input.ptr)[0]; } - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - if (((int16_t*)input->ptr)[index + i * N] > *max) - *max = ((int16_t*)input->ptr)[index + i * N]; + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int16_t*)input.ptr)[index + i * N] > *max) + *max = ((int16_t*)input.ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) getMax_16bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_32bit(struct c_array *input, ssize_t dim, ssize_t index, int32_t *max) { +int getMax_32bit(struct c_array input, ssize_t dim, ssize_t index, int32_t *max) { // to be initially called with getMax_32bit(input, 0, 0, *0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; if (dim == 0) { - if (input->shape[0] == 0) { + if (input.shape[0] == 0) { *max = 0; return 1; } - *max = ((int32_t*)input->ptr)[0]; + *max = ((int32_t*)input.ptr)[0]; } - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - if (((int32_t*)input->ptr)[index + i * N] > *max) - *max = ((int32_t*)input->ptr)[index + i * N]; + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int32_t*)input.ptr)[index + i * N] > *max) + *max = ((int32_t*)input.ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) getMax_32bit(input, dim + 1, index + i * N, max); return 1; } -int getMax_64bit(struct c_array *input, ssize_t dim, ssize_t index, int64_t *max) { +int getMax_64bit(struct c_array input, ssize_t dim, ssize_t index, int64_t *max) { // to be initially called with getMax_64bit(input, 0, 0, *0) - if (dim > input->ndim - 1) + if (dim > input.ndim - 1) return 0; if (dim == 0) { - if (input->shape[0] == 0) { + if (input.shape[0] == 0) { *max = 0; return 1; } - *max = ((int64_t*)input->ptr)[0]; + *max = ((int64_t*)input.ptr)[0]; } - ssize_t N = input->strides[dim] / input->itemsize; - if (dim == input->ndim - 1) { - for (ssize_t i = 0; i < input->shape[dim]; i++) - if (((int64_t*)input->ptr)[index + i * N] > *max) - *max = ((int64_t*)input->ptr)[index + i * N]; + ssize_t N = input.strides[dim] / input.itemsize; + if (dim == input.ndim - 1) { + for (ssize_t i = 0; i < input.shape[dim]; i++) + if (((int64_t*)input.ptr)[index + i * N] > *max) + *max = ((int64_t*)input.ptr)[index + i * N]; return 1; } - for (ssize_t i = 0; i < input->shape[dim]; i++) + for (ssize_t i = 0; i < input.shape[dim]; i++) getMax_64bit(input, dim + 1, index + i * N, max); return 1; } -int offsets2parents_8bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int8_t*)offsets->ptr)[i * N]) - ((int8_t*)parents->ptr)[j++] = (int8_t)k; +int offsets2parents_8bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int8_t*)offsets.ptr)[i * N]) + ((int8_t*)parents.ptr)[j++] = (int8_t)k; k++; } return 1; } -int offsets2parents_16bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int16_t*)offsets->ptr)[i * N]) - ((int16_t*)parents->ptr)[j++] = (int16_t)k; +int offsets2parents_16bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int16_t*)offsets.ptr)[i * N]) + ((int16_t*)parents.ptr)[j++] = (int16_t)k; k++; } return 1; } -int offsets2parents_32bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int32_t*)offsets->ptr)[i * N]) - ((int32_t*)parents->ptr)[j++] = (int32_t)k; +int offsets2parents_32bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int32_t*)offsets.ptr)[i * N]) + ((int32_t*)parents.ptr)[j++] = (int32_t)k; k++; } return 1; } -int offsets2parents_64bit(struct c_array *offsets, struct c_array *parents) { - ssize_t N = offsets->strides[0] / offsets->itemsize, j = 0, k = -1; - for (ssize_t i = 0; i < offsets->size; i++) { - while (j < (ssize_t)((int64_t*)offsets->ptr)[i * N]) - ((int64_t*)parents->ptr)[j++] = (int64_t)k; +int offsets2parents_64bit(struct c_array offsets, struct c_array parents) { + ssize_t N = offsets.strides[0] / offsets.itemsize, j = 0, k = -1; + for (ssize_t i = 0; i < offsets.size; i++) { + while (j < (ssize_t)((int64_t*)offsets.ptr)[i * N]) + ((int64_t*)parents.ptr)[j++] = (int64_t)k; k++; } return 1; } -int offsets2parents_CPU(struct c_array *offsets, struct c_array *parents) { +int offsets2parents_CPU(struct c_array offsets, struct c_array parents) { /* PURPOSE: - converts offsets to parents PREREQUISITES: @@ -263,127 +263,127 @@ int offsets2parents_CPU(struct c_array *offsets, struct c_array *parents) { - parents is a NEW array of size offsets[-1] - offsets and parents are of the same type */ - if (offsets->itemsize == 1) + if (offsets.itemsize == 1) return offsets2parents_8bit(offsets, parents); - if (offsets->itemsize == 2) + if (offsets.itemsize == 2) return offsets2parents_16bit(offsets, parents); - if (offsets->itemsize == 4) + if (offsets.itemsize == 4) return offsets2parents_32bit(offsets, parents); - if (offsets->itemsize == 8) + if (offsets.itemsize == 8) return offsets2parents_64bit(offsets, parents); return 0; } -int counts2offsets_8bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int8_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int8_t*)offsets->ptr)[i + 1] = ((int8_t*)offsets->ptr)[i] + ((int8_t*)counts->ptr)[i * N]; +int counts2offsets_8bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int8_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int8_t*)offsets.ptr)[i + 1] = ((int8_t*)offsets.ptr)[i] + ((int8_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_16bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int16_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int16_t*)offsets->ptr)[i + 1] = ((int16_t*)offsets->ptr)[i] + ((int16_t*)counts->ptr)[i * N]; +int counts2offsets_16bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int16_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int16_t*)offsets.ptr)[i + 1] = ((int16_t*)offsets.ptr)[i] + ((int16_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_32bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int32_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int32_t*)offsets->ptr)[i + 1] = ((int32_t*)offsets->ptr)[i] + ((int32_t*)counts->ptr)[i * N]; +int counts2offsets_32bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int32_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int32_t*)offsets.ptr)[i + 1] = ((int32_t*)offsets.ptr)[i] + ((int32_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_64bit(struct c_array *counts, struct c_array *offsets) { - ssize_t N = counts->strides[0] / counts->itemsize; - ((int64_t*)offsets->ptr)[0] = 0; - for (ssize_t i = 0; i < counts->size; i++) - ((int64_t*)offsets->ptr)[i + 1] = ((int64_t*)offsets->ptr)[i] + ((int64_t*)counts->ptr)[i * N]; +int counts2offsets_64bit(struct c_array counts, struct c_array offsets) { + ssize_t N = counts.strides[0] / counts.itemsize; + ((int64_t*)offsets.ptr)[0] = 0; + for (ssize_t i = 0; i < counts.size; i++) + ((int64_t*)offsets.ptr)[i + 1] = ((int64_t*)offsets.ptr)[i] + ((int64_t*)counts.ptr)[i * N]; return 1; } -int counts2offsets_CPU(struct c_array *counts, struct c_array *offsets) { +int counts2offsets_CPU(struct c_array counts, struct c_array offsets) { /* PURPOSE: - converts counts to offsets PREREQUISITES: - counts is a non-negative, 1d int array - - offsets is a NEW array of size offsets->size + 1 + - offsets is a NEW array of size offsets.size + 1 - counts and offsets are of the same type - note: for >1d counts, flatten it first */ - if (counts->itemsize == 1) + if (counts.itemsize == 1) return counts2offsets_8bit(counts, offsets); - if (counts->itemsize == 2) + if (counts.itemsize == 2) return counts2offsets_16bit(counts, offsets); - if (counts->itemsize == 4) + if (counts.itemsize == 4) return counts2offsets_32bit(counts, offsets); - if (counts->itemsize == 8) + if (counts.itemsize == 8) return counts2offsets_64bit(counts, offsets); return 0; } -int startsstops2parents_8bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { - for (ssize_t i = 0; i < parents->size; i++) - ((int8_t*)parents->ptr)[i] = -1; - ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; - for (ssize_t i = 0; i < starts->size; i++) - for (ssize_t j = (ssize_t)((int8_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int8_t*)stops->ptr)[i * N_sto]; j++) - ((int8_t*)parents->ptr)[j] = (int8_t)i; +int startsstops2parents_8bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int8_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int8_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int8_t*)stops.ptr)[i * N_sto]; j++) + ((int8_t*)parents.ptr)[j] = (int8_t)i; return 1; } -int startsstops2parents_16bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { - for (ssize_t i = 0; i < parents->size; i++) - ((int16_t*)parents->ptr)[i] = -1; - ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; - for (ssize_t i = 0; i < starts->size; i++) - for (ssize_t j = (ssize_t)((int16_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int16_t*)stops->ptr)[i * N_sto]; j++) - ((int16_t*)parents->ptr)[j] = (int16_t)i; +int startsstops2parents_16bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int16_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int16_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int16_t*)stops.ptr)[i * N_sto]; j++) + ((int16_t*)parents.ptr)[j] = (int16_t)i; return 1; } -int startsstops2parents_32bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { - for (ssize_t i = 0; i < parents->size; i++) - ((int32_t*)parents->ptr)[i] = -1; - ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; - for (ssize_t i = 0; i < starts->size; i++) - for (ssize_t j = (ssize_t)((int32_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int32_t*)stops->ptr)[i * N_sto]; j++) - ((int32_t*)parents->ptr)[j] = (int32_t)i; +int startsstops2parents_32bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int32_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int32_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int32_t*)stops.ptr)[i * N_sto]; j++) + ((int32_t*)parents.ptr)[j] = (int32_t)i; return 1; } -int startsstops2parents_64bit(struct c_array *starts, struct c_array *stops, struct c_array *parents) { - for (ssize_t i = 0; i < parents->size; i++) - ((int64_t*)parents->ptr)[i] = -1; - ssize_t N_sta = starts->strides[0] / starts->itemsize, N_sto = stops->strides[0] / stops->itemsize; - for (ssize_t i = 0; i < starts->size; i++) - for (ssize_t j = (ssize_t)((int64_t*)starts->ptr)[i * N_sta]; j < (ssize_t)((int64_t*)stops->ptr)[i * N_sto]; j++) - ((int64_t*)parents->ptr)[j] = (int64_t)i; +int startsstops2parents_64bit(struct c_array starts, struct c_array stops, struct c_array parents) { + for (ssize_t i = 0; i < parents.size; i++) + ((int64_t*)parents.ptr)[i] = -1; + ssize_t N_sta = starts.strides[0] / starts.itemsize, N_sto = stops.strides[0] / stops.itemsize; + for (ssize_t i = 0; i < starts.size; i++) + for (ssize_t j = (ssize_t)((int64_t*)starts.ptr)[i * N_sta]; j < (ssize_t)((int64_t*)stops.ptr)[i * N_sto]; j++) + ((int64_t*)parents.ptr)[j] = (int64_t)i; return 1; } -int startsstops2parents_CPU(struct c_array *starts, struct c_array *stops, struct c_array *parents) { +int startsstops2parents_CPU(struct c_array starts, struct c_array stops, struct c_array parents) { /* PURPOSE: - converts starts and stops to parents PREREQUISITES: - starts and stops are non-negative 1d int arrays - starts must be less than or equal to stops - starts must be of equal or shorter length than stops - - parents is a NEW array of length stops->maximum + - parents is a NEW array of length stops.maximum - starts, stops, and parents are of the same type - note: for >1d starts/stops, flatten them first */ - if (starts->itemsize == 1) + if (starts.itemsize == 1) return startsstops2parents_8bit(starts, stops, parents); - if (starts->itemsize == 2) + if (starts.itemsize == 2) return startsstops2parents_16bit(starts, stops, parents); - if (starts->itemsize == 4) + if (starts.itemsize == 4) return startsstops2parents_32bit(starts, stops, parents); - if (starts->itemsize == 8) + if (starts.itemsize == 8) return startsstops2parents_64bit(starts, stops, parents); return 0; } diff --git a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h index 221c2243..b370514f 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h +++ b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h @@ -8,63 +8,54 @@ namespace py = pybind11; -int py2c(py::array py, struct c_array *c) { - py::buffer_info info = py.request(); +struct c_array py2c(py::array input) { char format[8]; - strcpy(format, info.format.c_str()); - c->ptr = info.ptr; - c->itemsize = info.itemsize; - c->size = info.size; - c->format = format; - c->ndim = info.ndim; - c->shape = &info.shape[0]; - c->strides = &info.strides[0]; - return 1; + strcpy(format, input.request().format.c_str()); + struct c_array out = { + input.request().ptr, + input.request().itemsize, + input.request().size, + format, + input.request().ndim, + &input.request().shape[0], + &input.request().strides[0] + }; + return out; } int makeIntNative_CPU(py::array input) { - struct c_array array_struct; - py2c(input, &array_struct); - if (!checkInt_CPU(&array_struct)) { + if (!checkInt_CPU(py2c(input))) { throw std::invalid_argument("Argument must be an int array"); } - if (!makeNative_CPU(&array_struct)) { + if (!makeNative_CPU(py2c(input))) { throw std::invalid_argument("Error in cpu_methods.h::makeNative_CPU"); } return 1; } int getMax_CPU(py::array input, std::int8_t* max) { - struct c_array array_struct; - py2c(input, &array_struct); - if (!getMax_8bit(&array_struct, 0, 0, max)) { + if (!getMax_8bit(py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_8bit"); } return 1; } int getMax_CPU(py::array input, std::int16_t* max) { - struct c_array array_struct; - py2c(input, &array_struct); - if (!getMax_16bit(&array_struct, 0, 0, max)) { + if (!getMax_16bit(py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_16bit"); } return 1; } int getMax_CPU(py::array input, std::int32_t* max) { - struct c_array array_struct; - py2c(input, &array_struct); - if (!getMax_32bit(&array_struct, 0, 0, max)) { + if (!getMax_32bit(py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_32bit"); } return 1; } int getMax_CPU(py::array input, std::int64_t* max) { - struct c_array array_struct; - py2c(input, &array_struct); - if (!getMax_64bit(&array_struct, 0, 0, max)) { + if (!getMax_64bit(py2c(input), 0, 0, max)) { throw std::invalid_argument("Error in cpu_methods.h::getMax_64bit"); } return 1; diff --git a/awkward-cpp/awkward/cpp/array/jagged.h b/awkward-cpp/awkward/cpp/array/jagged.h index ae76cc1b..b3996a8d 100644 --- a/awkward-cpp/awkward/cpp/array/jagged.h +++ b/awkward-cpp/awkward/cpp/array/jagged.h @@ -313,12 +313,7 @@ class JaggedArray : public AwkwardArray { ssize_t parents_length = (ssize_t)offsets_ptr[(offsets_info.size - 1) * N]; auto parents = py::array_t(parents_length); - struct c_array offsets_struct; - py2c(offsets, &offsets_struct); - struct c_array parents_struct; - py2c(parents, &parents_struct); - - if (!offsets2parents_CPU(&offsets_struct, &parents_struct)) { + if (!offsets2parents_CPU(py2c(offsets), py2c(parents))) { throw std::invalid_argument("Error in cpu_methods.h::offsets2parents_CPU"); } return parents; @@ -333,13 +328,7 @@ class JaggedArray : public AwkwardArray { makeIntNative_CPU(counts); counts = counts.cast>(); auto offsets = py::array_t(counts.request().size + 1); - - struct c_array counts_struct; - py2c(counts, &counts_struct); - struct c_array offsets_struct; - py2c(offsets, &offsets_struct); - - if (!counts2offsets_CPU(&counts_struct, &offsets_struct)) { + if (!counts2offsets_CPU(py2c(counts), py2c(offsets))) { throw std::invalid_argument("Error in cpu_methods.h::counts2offsets_CPU"); } return offsets; @@ -360,14 +349,7 @@ class JaggedArray : public AwkwardArray { getMax_CPU(stops_, &max); auto parents = py::array_t((ssize_t)max); - struct c_array starts_struct; - py2c(starts_, &starts_struct); - struct c_array stops_struct; - py2c(stops_, &stops_struct); - struct c_array parents_struct; - py2c(parents, &parents_struct); - - if (!startsstops2parents_CPU(&starts_struct, &stops_struct, &parents_struct)) { + if (!startsstops2parents_CPU(py2c(starts_), py2c(stops_), py2c(parents))) { throw std::invalid_argument("Error in cpu_methods.h::startsstops2parents_CPU"); } return parents; From a35cf39ba49645c8f655e36e6fa1cc049786f07e Mon Sep 17 00:00:00 2001 From: EscottC Date: Tue, 16 Jul 2019 17:00:38 -0700 Subject: [PATCH 24/26] I think I found the problem, finally I'll still need to refine how I handle it, though. Right now I'm copying the strides and shape of each py::array to new arrays to be used in the CPU methods' structs --- awkward-cpp/awkward/cpp/array/cpu_pybind11.h | 27 ++++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h index b370514f..56ec331b 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h +++ b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h @@ -9,16 +9,27 @@ namespace py = pybind11; struct c_array py2c(py::array input) { - char format[8]; - strcpy(format, input.request().format.c_str()); + py::buffer_info info = input.request(); + + if (info.ndim > 15) { + throw std::invalid_argument("Array cannot exceed 15 dimensions"); + } + + char format[15]; + strcpy(format, info.format.c_str()); + ssize_t shape[15]; + std::copy(info.shape.begin(), info.shape.end(), shape); + ssize_t strides[15]; + std::copy(info.strides.begin(), info.strides.end(), strides); + struct c_array out = { - input.request().ptr, - input.request().itemsize, - input.request().size, + info.ptr, + info.itemsize, + info.size, format, - input.request().ndim, - &input.request().shape[0], - &input.request().strides[0] + info.ndim, + &shape[0], + &strides[0] }; return out; } From d9110702f9048b9252d5809a444f463c02889373 Mon Sep 17 00:00:00 2001 From: EscottC Date: Tue, 16 Jul 2019 17:34:16 -0700 Subject: [PATCH 25/26] a more efficient fix --- awkward-cpp/awkward/cpp/array/cpu_pybind11.h | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h index 56ec331b..b76ec782 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h +++ b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h @@ -11,16 +11,10 @@ namespace py = pybind11; struct c_array py2c(py::array input) { py::buffer_info info = input.request(); - if (info.ndim > 15) { - throw std::invalid_argument("Array cannot exceed 15 dimensions"); - } - char format[15]; strcpy(format, info.format.c_str()); - ssize_t shape[15]; - std::copy(info.shape.begin(), info.shape.end(), shape); - ssize_t strides[15]; - std::copy(info.strides.begin(), info.strides.end(), strides); + std::vector shape = info.shape; + std::vector strides = info.strides; struct c_array out = { info.ptr, From ffea73252e83de98b4d9dc1beff4e968902c39b3 Mon Sep 17 00:00:00 2001 From: EscottC Date: Tue, 16 Jul 2019 18:18:00 -0700 Subject: [PATCH 26/26] Revert "a more efficient fix" This reverts commit d9110702f9048b9252d5809a444f463c02889373. --- awkward-cpp/awkward/cpp/array/cpu_pybind11.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h index b76ec782..56ec331b 100644 --- a/awkward-cpp/awkward/cpp/array/cpu_pybind11.h +++ b/awkward-cpp/awkward/cpp/array/cpu_pybind11.h @@ -11,10 +11,16 @@ namespace py = pybind11; struct c_array py2c(py::array input) { py::buffer_info info = input.request(); + if (info.ndim > 15) { + throw std::invalid_argument("Array cannot exceed 15 dimensions"); + } + char format[15]; strcpy(format, info.format.c_str()); - std::vector shape = info.shape; - std::vector strides = info.strides; + ssize_t shape[15]; + std::copy(info.shape.begin(), info.shape.end(), shape); + ssize_t strides[15]; + std::copy(info.strides.begin(), info.strides.end(), strides); struct c_array out = { info.ptr,