diff --git a/CMakeLists.txt b/CMakeLists.txt index 3021648..1312c08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,10 @@ set(CMAKE_EXE_LINKER_FLAGS_INIT "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++") project(irods_rule_engine_plugin-python CXX) +include(GenerateExportHeader) +set(CMAKE_CXX_VISIBILITY_PRESET default) +set(CMAKE_VISIBILITY_INLINES_HIDDEN 0) + include(${IRODS_TARGETS_PATH}) set(IRODS_PLUGIN_VERSION "${IRODS_VERSION}.${IRODS_PLUGIN_REVISION}") @@ -90,7 +94,15 @@ set( irods_server ) - target_compile_definitions(${PLUGIN} PRIVATE ${IRODS_RULE_ENGINE_PYTHON_PLUGIN_COMPILE_DEFINITIONS} ${IRODS_COMPILE_DEFINITIONS}) + target_compile_definitions(${PLUGIN} PRIVATE ${IRODS_RULE_ENGINE_PYTHON_PLUGIN_COMPILE_DEFINITIONS} ${IRODS_COMPILE_DEFINITIONS} + RODS_SERVER + IRODS_QUERY_ENABLE_SERVER_SIDE_API + RODS_ENABLE_SYSLOG + IRODS_FILESYSTEM_ENABLE_SERVER_SIDE_API + IRODS_IO_TRANSPORT_ENABLE_SERVER_SIDE_API + SPDLOG_FMT_EXTERNAL + SPDLOG_NO_TLS + ) #target_compile_options(${PLUGIN} PRIVATE -Wno-write-strings) set_property(TARGET ${PLUGIN} PROPERTY CXX_STANDARD ${IRODS_CXX_STANDARD}) @@ -116,6 +128,8 @@ install( ${CMAKE_SOURCE_DIR}/core.py.template ${CMAKE_SOURCE_DIR}/session_vars.py ${CMAKE_SOURCE_DIR}/genquery.py + ${CMAKE_SOURCE_DIR}/irods_query_wrapper.py + ${CMAKE_SOURCE_DIR}/test_irods_query.py DESTINATION etc/irods ) diff --git a/irods_query_wrapper.py b/irods_query_wrapper.py new file mode 100644 index 0000000..03c8640 --- /dev/null +++ b/irods_query_wrapper.py @@ -0,0 +1,74 @@ +import irods_query as _irods_query +import re + +class IrodsQuery(object): + + class Row( object ): + def keys(self): return self.columns + def as_dict(self): return { k : self[k] for k in self.keys() } + def __init__(self,values,columns): + self.values = values + self.columns = columns + self.columns_dict = dict(zip(columns,values)) + def __iter__(self): + return iter(self.values) + def __getitem__(self,n): + if isinstance(n,int): return self.values[n] + elif isinstance(n,str): return self.columns_dict[n] + else: raise KeyError + + from irods_query import (query_iterator, vector_of_string) + + GENERAL = _irods_query.query_type.GENERAL + SPECIFIC = _irods_query.query_type.SPECIFIC + + __attributes = re.split('\s+', 'comm zone_hint query_string auto_fields query_type query_limit row_offset bind_args') + + def __init__(self, comm, # rei.rsComm from rule parameter + query_string, # "select FIELD1,FIELD2 from ... where ... " + auto_fields = (), # Up to n dictionary-like keys to correspond with FIELD(0..n-1) + zone_hint = "", + query_limit = 0, + row_offset = 0, + query_type = _irods_query.query_type.GENERAL, # IrodsQuery.GENERAL + bind_args = () ): # args for ? ; only if query_type is SPECIFIC + self.__qi = None + for x in self.__attributes: + setattr(self,x,locals()[x]) + + def __iter__(self): + if self.__qi is None: + self.__args = self.vector_of_string() + self.__args.assign(self.bind_args) + self.__qi = iter( self.query_iterator( self.comm, + self.query_string, + self.__args, "", 0, 0, self.query_type)) + return self + + def next(self): + try: + n = next( self.__qi ) + except StopIteration as e: + self.__qi = None + raise + else: + return self.Row(n, self.auto_fields) + + @property + def args(self): return tuple(self.__args) + +# -- usage in core.py -- +# +# from irods_query_wrapper import ( IrodsQuery ) +# +# def gen_q(arg,cbk,rei): +# for (i,d,c) in IrodsQuery (rei.rsComm, "select DATA_ID,DATA_NAME,COLL_NAME"): +# cbk.writeLine ("stdout", "id = %s : %s/%s" % (i,c,d) ) +# +# def spec_q(arg,cbk,rei): +# for row in IrodsQuery( rei.rsComm, "listGroupsForUser", +# bind_args=('dan',), +# auto_fields=('grp_id','grp_name'), +# query_type = IrodsQuery.SPECIFIC +# ): +# cbk.writeLine ("stdout", "Group ID {grp_id} NAME {grp_name}".format(**row)) diff --git a/irods_rule_engine_plugin-python.cxx b/irods_rule_engine_plugin-python.cxx index 221383d..2576d04 100644 --- a/irods_rule_engine_plugin-python.cxx +++ b/irods_rule_engine_plugin-python.cxx @@ -4,7 +4,9 @@ #include #include #include +#include #include +#include #include #include "boost/date_time.hpp" @@ -14,6 +16,9 @@ #include "boost/filesystem/operations.hpp" #include "boost/filesystem/operations.hpp" +#define IRODS_QUERY_ENABLE_SERVER_SIDE_API 1 +#include "irods_query.hpp" + #include "rodsErrorTable.h" #include "irods_error.hpp" #include "irods_re_plugin.hpp" @@ -28,6 +33,9 @@ #define register #include "boost/python/slice.hpp" +#include "boost/python/iterator.hpp" +#include "boost/python/stl_iterator.hpp" +#include "boost/python/enum.hpp" #include "boost/python/module_init.hpp" #undef register @@ -311,6 +319,48 @@ namespace } }; // struct CallbackWrapper + + template + void vector_assign(std::vector& l, bp::object o) { + // Turn a Python sequence into an STL input range + bp::stl_input_iterator begin(o), end; + l.assign(begin, end); + } + + using Qiter = irods::query; + using Qtype = Qiter::query_type; + + BOOST_PYTHON_MODULE(irods_query) + { + bp::enum_ ("query_type") .value("GENERAL", Qiter::GENERAL) + .value("SPECIFIC", Qiter::SPECIFIC) + .export_values(); + + bp::class_ qiterclass + ("query_iterator",bp::init*, // _specific_query_args + const std::string& , // _zone_hint (= "" default) + uintmax_t , // query limit (= 0 default) + uintmax_t , // row offset (= 0 default) + Qiter::query_type>()); // query type (GENERAL = 0, SPECIFIC = 1) + + qiterclass.def(bp::init()) // either query type but without bind args + .def(bp::init()) // query type (GENERAL = 0, SPECIFIC = 1) + .def("__iter__",bp::iterator()); + + using Vs = std::vector; + + bp::class_ >("vector_of_string") + .def("assign", &vector_assign) + .def("__getitem__",+[](const std::vector &obj, int i) {return obj.at(i);} ) + .def("__len__",&Vs::size); + } + BOOST_PYTHON_MODULE(plugin_wrappers) { bp::class_("RuleCallWrapper", bp::no_init) @@ -335,11 +385,14 @@ irods::error start(irods::default_re_ctx&, const std::string& _instance_name) PyImport_AppendInittab("plugin_wrappers", &initplugin_wrappers); PyImport_AppendInittab("irods_types", &initirods_types); PyImport_AppendInittab("irods_errors", &initirods_errors); + PyImport_AppendInittab("irods_query", &initirods_query); #else PyImport_AppendInittab("plugin_wrappers", &PyInit_plugin_wrappers); PyImport_AppendInittab("irods_types", &PyInit_irods_types); PyImport_AppendInittab("irods_errors", &PyInit_irods_errors); + PyImport_AppendInittab("irods_query", &PyInit_irods_query); #endif + Py_InitializeEx(0); std::lock_guard lock{python_mutex}; boost::filesystem::path full_path( boost::filesystem::current_path() ); @@ -355,6 +408,7 @@ irods::error start(irods::default_re_ctx&, const std::string& _instance_name) bp::object plugin_wrappers = bp::import("plugin_wrappers"); bp::object irods_types = bp::import("irods_types"); bp::object irods_errors = bp::import("irods_errors"); + bp::object irods_query = bp::import("irods_query"); StringFromPythonUnicode::register_converter(); } @@ -503,9 +557,12 @@ irods::error exec_rule(const irods::default_re_ctx&, bp::object core_namespace = core_module.attr("__dict__"); bp::object irods_types = bp::import("irods_types"); bp::object irods_errors = bp::import("irods_errors"); + bp::object irods_query = bp::import("irods_query"); core_namespace["irods_types"] = irods_types; core_namespace["irods_errors"] = irods_errors; + core_namespace["irods_types"] = irods_types; + core_namespace["irods_query"] = irods_query; bp::object rule_function = core_module.attr(rule_name.c_str()); @@ -635,6 +692,7 @@ irods::error exec_rule_text(const irods::default_re_ctx&, bp::object main_namespace = main_module.attr("__dict__"); bp::object irods_types = bp::import("irods_types"); bp::object irods_errors = bp::import("irods_errors"); + bp::object irods_query = bp::import("irods_query"); // Import global INPUT and OUTPUT variables main_namespace["global_vars"] = global_vars_python; @@ -642,6 +700,7 @@ irods::error exec_rule_text(const irods::default_re_ctx&, // Import global constants main_namespace["irods_types"] = irods_types; main_namespace["irods_errors"] = irods_errors; + main_namespace["irods_query"] = irods_query; // Parse input rule_text into useable Python fcns // Delete first line ("@external") @@ -744,6 +803,7 @@ irods::error exec_rule_expression(irods::default_re_ctx&, bp::object main_module = bp::import("__main__"); bp::object irods_types = bp::import("irods_types"); bp::object irods_errors= bp::import("irods_errors"); + bp::object irods_query = bp::import("irods_query"); bp::object main_namespace = main_module.attr("__dict__"); // Import global INPUT and OUTPUT variables @@ -752,6 +812,7 @@ irods::error exec_rule_expression(irods::default_re_ctx&, // Import globals main_namespace["irods_types"] = irods_types; main_namespace["irods_errors"] = irods_errors; + main_namespace["irods_query"] = irods_query; // Add def expressionFcn(rule_args, callback):\n to start of rule text std::string rule_name = "expressionFcn"; diff --git a/query_python_demo.sh b/query_python_demo.sh new file mode 100755 index 0000000..2bd5af6 --- /dev/null +++ b/query_python_demo.sh @@ -0,0 +1,14 @@ +#!/bin/sh +case $1 in + 0) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qc2gmain('1')" null ruleExecOut ;; + 1) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qc2gmain('0')" null ruleExecOut ;; + 2) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qc2dmain()" null ruleExecOut ;; + 3) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qc1main()" null ruleExecOut ;; + 4) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qc2main()" null ruleExecOut ;; + 5) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qcmain()" null ruleExecOut ;; + 6) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qmain()" null ruleExecOut ;; + 7) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qqmain('2')" null ruleExecOut ;; + 8) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qqmain('5')" null ruleExecOut ;; + 9) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qqmain('7')" null ruleExecOut ;; + *) exit 123;; +esac diff --git a/test_irods_query.py b/test_irods_query.py new file mode 100644 index 0000000..a519e51 --- /dev/null +++ b/test_irods_query.py @@ -0,0 +1,129 @@ + +import re +import os + +if not os.environ.get('IRODS__NOT_IMPORTING_FROM_RULEBASE'): + + from irods_query_wrapper import ( IrodsQuery ) + import irods_query as _irods_query + from irods_query import (query_iterator, vector_of_string, query_type) + from irods_query_wrapper import * # --> IrodsQuery + +__all__ = [ 'Test' ] + +class Test: + + @staticmethod + def qc2gmain(arg,cbk,rei): + + if arg[:1] == [ '0', ]: + + qGen = "select USER_GROUP_ID,USER_GROUP_NAME where USER_NAME = 'dan' and USER_GROUP_NAME != 'rodsgroup'" + q_with_fields = IrodsQuery( rei.rsComm, qGen, auto_fields = ('gpid','gpnm'), query_type_ = query_type.GENERAL ) + for row in q_with_fields: + cbk.writeLine('stderr', repr(row.as_dict())) + + elif arg[:1] == [ '1', ]: + + q = IrodsQuery( rei.rsComm, 'select COLL_NAME,DATA_NAME', query_type_ = query_type.GENERAL ) + for colldata in q: + cbk.writeLine('stderr','{0}/{1}'.format(*colldata)) + + @staticmethod + def qc2dmain(arg,cbk,rei): + i = IrodsQuery( rei.rsComm, 'listGroupsForUser', auto_fields = ('GrpID','GrpName'), bind_args = ('dan',)) + for z in i: + cbk.writeLine('stderr','id={GrpID}. name={GrpName}.'.format(**z)) + + @staticmethod + def qc2main(arg,cbk,rei): + i = IrodsQuery( rei.rsComm, 'listGroupsForUser', bind_args = ('dan',)) + for x,y in i: + cbk.writeLine('stderr','{x},{y}'.format(**locals())) + + @staticmethod + def qc1main(arg,cbk,rei): + i = IrodsQuery( rei.rsComm, 'sq1y1', bind_args = ('dan',)) + for x, in i: + cbk.writeLine('stderr','{x}, '.format(**locals())) + + @staticmethod + def qcmain(arg,cbk,rei): + i = IrodsQuery( rei.rsComm, 'sq2arg', bind_args = ('dan','rodsgroup')) + for x,y in i: + cbk.writeLine('stderr','{x}, {y}'.format(**locals())) + + + ''' + sq1y1 = (""" + select group_user_id from R_USER_GROUP ug inner join R_USER_MAIN u on ug.group_user_id = u.user_id + where user_type_name = 'rodsgroup' and ug.user_id = ( + select user_id from R_USER_MAIN where user_name = ? and user_type_name != 'rodsgroup') + """) + #--------------------------------- + sq1 = (""" + select group_user_id, user_name from R_USER_GROUP ug inner join R_USER_MAIN u on ug.group_user_id = u.user_id + where user_type_name = 'rodsgroup' and ug.user_id = + (select user_id from R_USER_MAIN where user_name = 'dan' and user_type_name != 'rodsgroup') + """) + #---- + sq2arg = (""" + select group_user_id, user_name from R_USER_GROUP ug inner join R_USER_MAIN u on ug.group_user_id = u.user_id + where user_type_name = 'rodsgroup' and ug.user_id = ( + select user_id from R_USER_MAIN where user_name = ? and user_type_name != ?) + """) + ''' + + @staticmethod + def qmain(arg,cbk,rei): + args = vector_of_string(); + args.assign( ['dan','rodsgroup'] ) + qit = query_iterator( rei.rsComm, + 'sq2arg', + args, "", 0, 0, _irods_query.query_type.SPECIFIC ) + qi = iter(qit) + try: + while True: + y = next(qi) + len(y) + cbk.writeLine('stderr','id = {y[0]} ; name = {y[1]}'.format(**locals()) ) + except StopIteration: + print('--stopped--') + + @staticmethod + def qqmain(cbk,rei,**kw): + q_general = "select USER_GROUP_ID, USER_GROUP_NAME where USER_NAME = 'dan' and USER_GROUP_NAME != 'rodsgroup'" + q_specific = "select group_user_id, user_name from R_USER_GROUP ug inner join R_USER_MAIN u on ug.group_user_id" \ + " = u.user_id where user_type_name = 'rodsgroup' and ug.user_id = (select " \ + "user_id from R_USER_MAIN where user_name = ? and user_type_name != 'rodsgroup')" + args = vector_of_string(); + empty = vector_of_string(); + args.assign( ['dan','rodsgroup'] ) + n_arg = kw['case_'] + if n_arg == "2": + # -- general query + qi = query_iterator( rei.rsComm, q_general ) + elif n_arg == "5": + # -- specific but no args passed + # test with: iadmin asq "" sq1 where sql like above but "?" replaced by eg "'username'" + qi = query_iterator( rei.rsComm, "sq1", 0, 0, _irods_query.query_type.SPECIFIC ) + elif n_arg == "7": + # test with: iadmin asq "" sq2arg where sql like above but "'rodsadmin'" replaced by "?" + qi = query_iterator( rei.rsComm, + 'sq2arg', + args, "", 0, 0, _irods_query.query_type.SPECIFIC ) + elif n_arg == "7g": + qi = query_iterator( rei.rsComm, + 'select DATA_ID', + empty, "", 0, 0, _irods_query.query_type.GENERAL ) + for _ in qi: + cbk.writeLine('stderr',(_)[0]) + return + for y in qi: + cbk.writeLine('stderr','id = {y[0]} ; name = {y[1]}'.format(**locals()) ) + + @staticmethod + def test_all_cases_in_qqmain(arg,cbk,rei): + for case in '2','5','7','7g': + Test.qqmain(cbk, rei, case_ = case) +