Add timedelta, timedelta64 and datetime64 plus respective conversions

Project.toml

@@ -16,7 +16,6 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 UnsafePointers = "e17b2a0c-0bdf-430a-bd0c-3a23cae4ff39"
 
 [compat]
-Aqua = "0 - 999"
 CondaPkg = "0.2.23"
 Dates = "1"
 Libdl = "1"
@@ -26,15 +25,5 @@ Pkg = "1"
 Requires = "1"
 Serialization = "1"
 Tables = "1"
-Test = "1"
-TestItemRunner = "0 - 999"
 UnsafePointers = "1"
 julia = "1.6.1"
-
-[extras]
-Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
-
-[targets]
-test = ["Aqua", "Test", "TestItemRunner"]

src/Convert/Convert.jl

@@ -45,9 +45,17 @@ using ..Core:
     pythrow,
     pybool_asbool
 using Dates: Date, Time, DateTime, Second, Millisecond, Microsecond, Nanosecond
+using Dates: Year, Month, Day, Hour, Minute, Week, Period, CompoundPeriod, canonicalize
 
 import ..Core: pyconvert
 
+# patch conversion to Period types for julia <= 1.7
+@static if VERSION < v"1.8.0-"
+    for T in (:Year, :Month, :Week, :Day, :Hour, :Minute, :Second, :Millisecond, :Microsecond, :Nanosecond)
+        @eval Base.convert(::Type{$T}, x::CompoundPeriod) = sum($T, x.periods; init = zero($T))
+    end
+end
+
 include("pyconvert.jl")
 include("rules.jl")
 include("ctypes.jl")

src/Convert/numpy.jl

@@ -1,3 +1,5 @@
+const CANONICALIZE_TIMEDELTA64 = Ref(false)
+
 struct pyconvert_rule_numpysimplevalue{R,S} <: Function end
 
 function (::pyconvert_rule_numpysimplevalue{R,SAFE})(::Type{T}, x::Py) where {R,SAFE,T}
@@ -27,6 +29,100 @@ const NUMPY_SIMPLE_TYPES = [
     ("complex128", ComplexF64),
 ]
 
+function pydatetime64(
+    _year::Integer=0, _month::Integer=1, _day::Integer=1, _hour::Integer=0, _minute::Integer=0,_second::Integer=0, _millisecond::Integer=0, _microsecond::Integer=0, _nanosecond::Integer=0;
+    year::Integer=_year, month::Integer=_month, day::Integer=_day, hour::Integer=_hour, minute::Integer=_minute, second::Integer=_second,
+    millisecond::Integer=_millisecond, microsecond::Integer=_microsecond, nanosecond::Integer=_nanosecond
+)
+    pyimport("numpy").datetime64("$(DateTime(year, month, day, hour, minute, second))") + 
+        pytimedelta64(; milliseconds = millisecond, microseconds = microsecond, nanoseconds = nanosecond)
+end
+function pydatetime64(@nospecialize(x::T)) where T <: Period
+    T <: Union{Week, Day, Hour, Minute, Second, Millisecond, Microsecond} || 
+        error("Unsupported Period type: `$x::$T`. Consider using pytimedelta64 instead.")
+    args = map(Base.Fix1(isa, x), (Day, Second, Millisecond, Microsecond,  Minute, Hour, Week))
+    pydatetime64(map(Base.Fix1(*, x.value), args)...)
+end
+function pydatetime64(x::Union{Date, DateTime})
+    pyimport("numpy").datetime64("$x")
+end
+export pydatetime64
+
+function pytimedelta64(
+    _years::Union{Nothing,Integer}=nothing, _months::Union{Nothing,Integer}=nothing,
+    _days::Union{Nothing,Integer}=nothing, _hours::Union{Nothing,Integer}=nothing,
+    _minutes::Union{Nothing,Integer}=nothing, _seconds::Union{Nothing,Integer}=nothing,
+    _milliseconds::Union{Nothing,Integer}=nothing, _microseconds::Union{Nothing,Integer}=nothing,
+    _nanoseconds::Union{Nothing,Integer}=nothing, _weeks::Union{Nothing,Integer}=nothing;
+    years::Union{Nothing,Integer}=_years, months::Union{Nothing,Integer}=_months,
+    days::Union{Nothing,Integer}=_days, hours::Union{Nothing,Integer}=_hours,
+    minutes::Union{Nothing,Integer}=_minutes, seconds::Union{Nothing,Integer}=_seconds,
+    milliseconds::Union{Nothing,Integer}=_milliseconds, microseconds::Union{Nothing,Integer}=_microseconds,
+    nanoseconds::Union{Nothing,Integer}=_nanoseconds, weeks::Union{Nothing,Integer}=_weeks,
+    canonicalize::Bool = false)
+
+    y::Integer = something(years, 0)
+    m::Integer = something(months, 0)
+    d::Integer = something(days, 0)
+    h::Integer = something(hours, 0)
+    min::Integer = something(minutes, 0)
+    s::Integer = something(seconds, 0)
+    ms::Integer = something(milliseconds, 0)
+    µs::Integer = something(microseconds, 0)
+    ns::Integer = something(nanoseconds, 0)
+    w::Integer = something(weeks, 0)
+    cp = sum((
+        Year(y), Month(m), Week(w), Day(d), Hour(h), Minute(min), Second(s), Millisecond(ms), Microsecond(µs), Nanosecond(ns))
+    )
+    # make sure the correct unit is used when value is 0
+    if isempty(cp.periods)
+        Units = (Second, Year, Month, Week, Day, Hour, Minute, Second, Millisecond, Microsecond, Nanosecond)
+        index::Integer = findlast(!isnothing, (0, years, months, weeks, days, hours, minutes, seconds, milliseconds, microseconds, nanoseconds));
+        pytimedelta64(Units[index](0))
+    else
+        pytimedelta64(cp; canonicalize)
+    end
+end
+function pytimedelta64(@nospecialize(x::T); canonicalize::Bool = false) where T <: Period
+    canonicalize && return pytimedelta64(@__MODULE__().canonicalize(x))
+
+    index = findfirst(T .== (Year, Month, Week, Day, Hour, Minute, Second, Millisecond, Microsecond, Nanosecond, T))::Int
+    unit = ("Y", "M", "W", "D", "h", "m", "s", "ms", "us", "ns", "")[index]
+    pyimport("numpy").timedelta64(x.value, unit)
+end
+function pytimedelta64(x::CompoundPeriod; canonicalize::Bool = false)
+    canonicalize && (x = @__MODULE__().canonicalize(x))
+    isempty(x.periods) ? pytimedelta64(Second(0)) : sum(pytimedelta64.(x.periods))
+end
+function pytimedelta64(x::Integer)
+    pyimport("numpy").timedelta64(x)
+end
+export pytimedelta64
+
+function pyconvert_rule_datetime64(::Type{DateTime}, x::Py)
+    unit, count = pyconvert(Tuple, pyimport("numpy").datetime_data(x))
+    value = reinterpret(Int64, pyconvert(Vector, x))[1]
+    units = ("Y", "M", "W", "D", "h", "m", "s", "ms", "us", "ns")
+    types = (Year, Month, Week, Day, Hour, Minute, Second, Millisecond, Microsecond, Nanosecond)
+    T = types[findfirst(==(unit), units)::Int]
+    pyconvert_return(DateTime(_base_datetime) + T(value * count))
+end
+
+function pyconvert_rule_timedelta64(::Type{CompoundPeriod}, x::Py)
+    unit, count = pyconvert(Tuple, pyimport("numpy").datetime_data(x))
+    value = reinterpret(Int64, pyconvert(Vector, x))[1]
+    units = ("Y", "M", "W", "D", "h", "m", "s", "ms", "us", "ns")
+    types = (Year, Month, Week, Day, Hour, Minute, Second, Millisecond, Microsecond, Nanosecond)
+    T = types[findfirst(==(unit), units)::Int]
+    cp = CompoundPeriod(T(value * count))
+    CANONICALIZE_TIMEDELTA64[] && (cp = @__MODULE__().canonicalize(cp))
+    pyconvert_return(cp)
+end
+
+function pyconvert_rule_timedelta64(::Type{T}, x::Py) where T<:Period
+    pyconvert_return(convert(T, pyconvert_rule_timedelta64(CompoundPeriod, x)))
+end
+
 function init_numpy()
     for (t, T) in NUMPY_SIMPLE_TYPES
         isbool = occursin("bool", t)
@@ -54,4 +150,14 @@ function init_numpy()
         iscomplex && pyconvert_add_rule(name, Complex, rule)
         isnumber && pyconvert_add_rule(name, Number, rule)
     end
+
+    priority = PYCONVERT_PRIORITY_ARRAY
+    pyconvert_add_rule("numpy:datetime64", DateTime, pyconvert_rule_datetime64, priority)
+    let TT = (CompoundPeriod, Year, Month, Day, Hour, Minute, Second, Millisecond, Microsecond, Nanosecond, Week)
+        Base.Cartesian.@nexprs 11 i -> pyconvert_add_rule("numpy:timedelta64", TT[i], pyconvert_rule_timedelta64, priority)
+    end
+
+    priority = PYCONVERT_PRIORITY_CANONICAL
+    pyconvert_add_rule("numpy:datetime64", DateTime, pyconvert_rule_datetime64, priority)
+    pyconvert_add_rule("numpy:timedelta64", Nanosecond, pyconvert_rule_timedelta, priority)    
 end

src/Convert/pyconvert.jl

@@ -428,6 +428,12 @@ function init_pyconvert()
         pyimport("collections.abc" => ("Iterable", "Sequence", "Set", "Mapping"))...,
     )
 
+    priority = PYCONVERT_PRIORITY_ARRAY
+    pyconvert_add_rule("datetime:datetime", DateTime, pyconvert_rule_datetime, priority)
+    for T in (Millisecond, Second, Nanosecond, Day, Hour, Minute, Second, Millisecond, Week, CompoundPeriod)
+        pyconvert_add_rule("datetime:timedelta", T, pyconvert_rule_timedelta, priority)
+    end
+
     priority = PYCONVERT_PRIORITY_CANONICAL
     pyconvert_add_rule("builtins:NoneType", Nothing, pyconvert_rule_none, priority)
     pyconvert_add_rule("builtins:bool", Bool, pyconvert_rule_bool, priority)

src/Convert/rules.jl

@@ -512,3 +512,16 @@ function pyconvert_rule_timedelta(::Type{Second}, x::Py)
     end
     return Second(days * 3600 * 24 + seconds)
 end
+
+function pyconvert_rule_timedelta(::Type{<:CompoundPeriod}, x::Py)
+    days = pyconvert(Int, x.days)
+    seconds = pyconvert(Int, x.seconds)
+    microseconds = pyconvert(Int, x.microseconds)
+    nanoseconds = pyhasattr(x, "nanoseconds") ? pyconvert(Int, x.nanoseconds) : 0
+    timedelta = Day(days) + Second(seconds) + Microsecond(microseconds) + Nanosecond(nanoseconds)
+    return pyconvert_return(timedelta)
+end
+
+function pyconvert_rule_timedelta(::Type{T}, x::Py) where T<:Period
+    pyconvert_return(convert(T, pyconvert_rule_timedelta(CompoundPeriod, x)))
+end

src/Core/Core.jl

@@ -25,7 +25,17 @@ using Dates:
     second,
     millisecond,
     microsecond,
-    nanosecond
+    nanosecond,
+    Day,
+    Hour,
+    Week,
+    Minute,
+    Second,
+    Millisecond,
+    Microsecond,
+    Period,
+    CompoundPeriod,
+    canonicalize
 using MacroTools: MacroTools, @capture
 using Markdown: Markdown
 

src/Core/Py.jl

@@ -158,6 +158,7 @@ Py(
 Py(x::Date) = pydate(x)
 Py(x::Time) = pytime(x)
 Py(x::DateTime) = pydatetime(x)
+Py(x::Union{Period, CompoundPeriod}) = pytimedelta(x)
 
 Base.string(x::Py) = pyisnull(x) ? "<py NULL>" : pystr(String, x)
 Base.print(io::IO, x::Py) = print(io, string(x))

src/Core/builtins.jl

@@ -1167,6 +1167,24 @@ end
 pydatetime(x::Date) = pydatetime(year(x), month(x), day(x))
 export pydatetime
 
+function pytimedelta(
+    _days::Int=0, _seconds::Int=0, _microseconds::Int=0, _milliseconds::Int=0, _minutes::Int=0, _hours::Int=0, _weeks::Int=0;
+    days::Int=_days, seconds::Int=_seconds, microseconds::Int=_microseconds, milliseconds::Int=_milliseconds, minutes::Int=_minutes, hours::Int=_hours, weeks::Int=_weeks
+)
+    pyimport("datetime").timedelta(days, seconds, microseconds, milliseconds, minutes, hours, weeks)
+end
+function pytimedelta(@nospecialize(x::T)) where T <: Period
+    T <: Union{Week, Day, Hour, Minute, Second, Millisecond, Microsecond} || 
+        error("Unsupported Period type: ", "Year, Month and Nanosecond are not supported, consider using pytimedelta64 instead.")
+    args = T .== (Day, Second, Microsecond, Millisecond, Minute, Hour, Week)
+    pytimedelta(x.value .* args...)
+end
+function pytimedelta(x::CompoundPeriod)
+    x =  canonicalize(x)
+    isempty(x.periods) ? pytimedelta(Second(0)) : sum(pytimedelta.(x.periods))
+end
+export pytimedelta
+
 function pytime_isaware(x)
     tzinfo = pygetattr(x, "tzinfo")
     if pyisnone(tzinfo)

src/PyMacro/PyMacro.jl

@@ -886,6 +886,9 @@ For example:
 - `import x: f as g` is translated to `g = pyimport("x" => "f")` (`from x import f as g` in Python)
 
 Compound statements such as `begin`, `if`, `while` and `for` are supported.
+Import statements are supported, e.g.
+- `import foo, bar`
+- `from os.path import join as py_joinpath, exists`
 
 See the online documentation for more details.
 
@@ -895,6 +898,19 @@ See the online documentation for more details.
 macro py(ex)
     esc(py_macro(ex, __module__, __source__))
 end
+
+macro py(keyword, modulename, ex)
+    keyword == :from || return :( nothing )
+
+    d = Dict(isa(a.args[1], Symbol) ? a.args[1] => a.args[1] : a.args[1].args[1] => a.args[2]  for a in ex.args)
+    vars = Expr(:tuple, values(d)...)
+    imports = Tuple(keys(d))
+
+    esc(quote
+        $vars = pyimport($(string(modulename)) => $(string.(imports)))
+    end)
+end
+
 export @py
 
 end

test/Compat.jl

@@ -79,11 +79,14 @@ end
 end
 
 @testitem "Tables.jl" begin
+    using CondaPkg
+    CondaPkg.add("pandas")
+
     @testset "pytable" begin
         x = (x = [1, 2, 3], y = ["a", "b", "c"])
         # pandas
-        # TODO: install pandas and test properly
-        @test_throws PyException pytable(x, :pandas)
+        t = pytable(x, :pandas)
+        @test pyconvert.(Int, Tuple(t.shape)) == (3, 2)
         # columns
         y = pytable(x, :columns)
         @test pyeq(Bool, y, pydict(x = [1, 2, 3], y = ["a", "b", "c"]))

test/CondaPkg.toml

@@ -0,0 +1,2 @@
+[deps]
+pandas = ""

test/Convert.jl

@@ -305,6 +305,59 @@ end
     @test_throws Exception pyconvert(Second, td(microseconds = 1000))
 end
 
+@testitem "timedelta64" begin
+    using Dates
+    using CondaPkg
+    CondaPkg.add("pandas")
+    using DataFrames
+
+    dt1 = pytimedelta(seconds = 1)
+    dt2 = pytimedelta64(seconds = 1)
+    @test pyeq(Bool, dt1, dt2)
+
+    @test pyeq(Bool, pytimedelta64(seconds = 10), pyimport("numpy").timedelta64(10, "s"))
+    @test pyeq(Bool, pytimedelta64(years = 10), pyimport("numpy").timedelta64(10, "Y"))
+    @test_throws Exception pytimedelta64(years = 10, seconds = 1)
+
+    @testset for x in [
+        -1_000_000_000,
+        -1_000_000,
+        -1_000,
+        -1,
+        0,
+        1,
+        1_000,
+        1_000_000,
+        1_000_000_000,
+    ], (Unit, unit) in [
+        (Nanosecond, :nanoseconds),
+        (Microsecond, :microseconds),
+        (Millisecond, :milliseconds),
+        (Second, :seconds),
+        (Minute, :minutes),
+        (Hour, :hours),
+        (Day, :days),
+        (Week, :weeks),
+        (Month, :months),
+        (Year, :years),
+    ]
+        y = pyconvert(Unit, pytimedelta64(; [unit => x]...))
+        @test y === Unit(x)
+    end
+    @test_throws Exception pyconvert(Second, td(microseconds = 1000))
+
+    jdf = DataFrame(x = [now() + Second(rand(1:1000)) for _ in 1:100], y = [Second(n) for n in 1:100])
+    pdf = pytable(jdf)
+    @test ispy(pdf.y)
+    @test pyeq(Bool, pdf.y[0], pytimedelta64(seconds = 1))
+    # automatic conversion from pytimedelta64 converts to Dates.CompoundPeriod
+    jdf2 = DataFrame(PyPandasDataFrame(pdf))
+    @test eltype(jdf2.y) == Dates.CompoundPeriod
+    # convert y column back to Seconds
+    jdf2.y = convert.(Second, jdf2.y)
+    @test pyeq(Bool, jdf, jdf2)
+end
+
 @testitem "pyconvert_add_rule (#364)" begin
     id = string(rand(UInt128), base = 16)
     pyexec(