WIP - C Data Interface

.gitignore

@@ -5,4 +5,4 @@ Manifest.toml
 *.jl.mem
 
 test/_scrap.jl
-.DS_STORE
+.vscode

Project.toml

@@ -1,7 +1,7 @@
 name = "Arrow"
 uuid = "69666777-d1a9-59fb-9406-91d4454c9d45"
 authors = ["quinnj <quinn.jacobd@gmail.com>"]
-version = "1.3.0"
+version = "1.4.1"
 
 [deps]
 BitIntegers = "c3b6d118-76ef-56ca-8cc7-ebb389d030a1"
@@ -33,6 +33,8 @@ JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
+Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
 
 [targets]
-test = ["Test", "Random", "JSON3", "StructTypes", "CategoricalArrays"]
+test = ["Test", "Random", "JSON3", "StructTypes", "CategoricalArrays", "PyCall", "Conda"]

src/Arrow.jl

@@ -92,7 +92,8 @@ include("arraytypes/arraytypes.jl")
 include("eltypes.jl")
 include("table.jl")
 include("write.jl")
-include("cinterface.jl")
+include("append.jl")
+include("CDataInterface/CDataInterface.jl")
 
 const LZ4_FRAME_COMPRESSOR = LZ4FrameCompressor[]
 const ZSTD_COMPRESSOR = ZstdCompressor[]
@@ -106,6 +107,7 @@ function __init__()
         CodecLz4.TranscodingStreams.initialize(lz4)
         push!(LZ4_FRAME_COMPRESSOR, lz4)
     end
+    OBJ_METADATA_LOCK[] = ReentrantLock()
     return
 end
 

src/ArrowTypes/src/ArrowTypes.jl

@@ -208,7 +208,8 @@ _symbol(ptr, len) = ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Int), ptr, len
 fromarrow(::Type{Symbol}, ptr::Ptr{UInt8}, len::Int) = _symbol(ptr, len)
 
 ArrowKind(::Type{<:AbstractArray}) = ListKind()
-fromarrow(::Type{A}, x::AbstractVector{T}) where {A <: AbstractVector{T}} where {T} = x
+fromarrow(::Type{A}, x::A) where {A <: AbstractVector{T}} where {T} = x
+fromarrow(::Type{A}, x::AbstractVector{T}) where {A <: AbstractVector{T}} where {T} = convert(A, x)
 ArrowKind(::Type{<:AbstractSet}) = ListKind()
 ArrowType(::Type{T}) where {T <: AbstractSet{S}} where {S} = Vector{S}
 toarrow(x::AbstractSet) = collect(x)

src/CDataInterface/CDataInterface.jl

@@ -0,0 +1,38 @@
+module CDataInterface
+
+import ..Arrow
+include("c_definitions.jl")
+include("jl_definitions.jl")
+include("c_to_jl.jl")
+
+export ArrowSchema, ArrowArray, getschema, getarray
+
+function get_schema(f)
+    schema_ref = Ref{CArrowSchema}()
+    ptr = Base.unsafe_convert(Ptr{CArrowSchema}, schema_ref)
+    f(ptr)
+    sch = ArrowSchema(schema_ref)
+    finalizer(sch) do x
+        r = getfield(x.c_arrow_schema[], :release)
+        if r != C_NULL
+            ccall(r, Cvoid, (Ptr{CArrowSchema},), x.carrowschema)
+        end
+    end
+    return sch
+end
+
+function get_array(f)
+    arr_ref = Ref{CArrowArray}()
+    ptr = Base.unsafe_convert(Ptr{CArrowArray}, arr_ref)
+    f(ptr)
+    arr = InterimCArrowArray(arr_ref)
+    finalizer(arr) do x
+        r = getfield(x.c_arrow_array[], :release)
+        if r != C_NULL
+            ccall(r, Cvoid, (Ptr{CArrowArray},), x.c_arrow_array)
+        end
+    end
+    return arr
+end
+
+end # module

src/cinterface.jl → src/CDataInterface/c_definitions.jl

@@ -1,7 +1,3 @@
-module CData
-
-export ArrowSchema, ArrowArray, getschema, getarray
-
 const ARROW_FLAG_DICTIONARY_ORDERED = 1
 const ARROW_FLAG_NULLABLE = 2
 const ARROW_FLAG_MAP_KEYS_SORTED = 4
@@ -22,7 +18,7 @@ CArrowSchema() = CArrowSchema(C_NULL, C_NULL, C_NULL, 0, 0, C_NULL, C_NULL, _CNU
 
 Base.propertynames(::CArrowSchema) = (:format, :name, :metadata, :flags, :n_children, :children, :dictionary)
 
-function readmetadata(ptr::Ptr{UInt8})
+function read_c_arrow_schema_metadata(ptr::Ptr{UInt8})
     pos = 1
     meta = Dict{String, String}()
     if ptr != C_NULL
@@ -49,7 +45,7 @@ function Base.getproperty(x::CArrowSchema, nm::Symbol)
     elseif nm === :name
         return unsafe_string(getfield(x, :name))
     elseif nm === :metadata
-        return readmetadata(getfield(x, :metadata))
+        return read_c_arrow_schema_metadata(getfield(x, :metadata))
     elseif nm === :flags
         return getfield(x, :flags)
     elseif nm === :n_children
@@ -64,34 +60,6 @@ function Base.getproperty(x::CArrowSchema, nm::Symbol)
     error("unknown property requested: $nm")
 end
 
-mutable struct ArrowSchema
-    format::String
-    name::String
-    metadata::Dict{String, String}
-    flags::Int64
-    n_children::Int64
-    children::Vector{ArrowSchema}
-    dictionary::Union{Nothing, ArrowSchema}
-    carrowschema::Ref{CArrowSchema}
-end
-
-ArrowSchema(s::Ref{CArrowSchema}) = ArrowSchema(s[].format, s[].name, s[].metadata, s[].flags, s[].n_children, map(ArrowSchema, s[].children), s[].dictionary === nothing ? nothing : ArrowSchema(s[].dictionary), s)
-ArrowSchema(s::CArrowSchema) = ArrowSchema(s.format, s.name, s.metadata, s.flags, s.n_children, map(ArrowSchema, s.children), s.dictionary === nothing ? nothing : ArrowSchema(s.dictionary), Ref{CArrowSchema}())
-
-function getschema(f)
-    schref = Ref{CArrowSchema}()
-    ptr = Base.unsafe_convert(Ptr{CArrowSchema}, schref)
-    f(ptr)
-    sch = ArrowSchema(schref)
-    finalizer(sch) do x
-        r = getfield(x.carrowschema[], :release)
-        if r != C_NULL
-            ccall(r, Cvoid, (Ptr{CArrowSchema},), x.carrowschema)
-        end
-    end
-    return sch
-end
-
 struct CArrowArray
     length::Int64
     null_count::Int64
@@ -131,35 +99,4 @@ function Base.getproperty(x::CArrowArray, nm::Symbol)
         return d == C_NULL ? nothing : unsafe_load(d)
     end
     error("unknown property requested: $nm")
-end
-
-mutable struct ArrowArray
-    length::Int64
-    null_count::Int64
-    offset::Int64
-    n_buffers::Int64
-    n_children::Int64
-    buffers::Vector{Ptr{UInt8}}
-    children::Vector{ArrowArray}
-    dictionary::Union{Nothing, ArrowArray}
-    carrowarray::Ref{CArrowArray}
-end
-
-ArrowArray(a::Ref{CArrowArray}) = ArrowArray(a[].length, a[].null_count, a[].offset, a[].n_buffers, a[].n_children, a[].buffers, map(ArrowArray, a[].children), a[].dictionary === nothing ? nothing : ArrowArray(a[].dictionary), a)
-ArrowArray(a::CArrowArray) = ArrowArray(a.length, a.null_count, a.offset, a.n_buffers, a.n_children, a.buffers, map(ArrowArray, a.children), a.dictionary === nothing ? nothing : ArrowArray(a.dictionary), Ref{CArrowArray}())
-
-function getarray(f)
-    arrref = Ref{CArrowArray}()
-    ptr = Base.unsafe_convert(Ptr{CArrowArray}, arrref)
-    f(ptr)
-    arr = ArrowArray(arrref)
-    finalizer(arr) do x
-        r = getfield(x.carrowarray[], :release)
-        if r != C_NULL
-            ccall(r, Cvoid, (Ptr{CArrowArray},), x.carrowarray)
-        end
-    end
-    return arr
-end
-
-end # module
+end