From 2e13cf0dab4833d471d8f6df0fe1e6809010536f Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Wed, 15 Apr 2020 17:01:28 +0100 Subject: [PATCH 001/341] Ensure we relinquish the GIL after initialising Python and have it when destroying callables --- src/runner/pythonrunner.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 3762f965..9cc65d19 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -733,6 +733,11 @@ struct Functor0_converter FunctorWrapper(boost::python::object callable) : m_Callable(callable) { } + ~FunctorWrapper() { + GILock lock; + m_Callable = bpy::object(); + } + RET operator()() { GILock lock; return (RET) m_Callable(); @@ -774,6 +779,11 @@ struct Functor1_converter FunctorWrapper(boost::python::object callable) : m_Callable(callable) { } + ~FunctorWrapper() { + GILock lock; + m_Callable = bpy::object(); + } + RET operator()(const PAR1 ¶m1) { GILock lock; return (RET) m_Callable(param1); @@ -815,6 +825,11 @@ struct Functor2_converter FunctorWrapper(boost::python::object callable) : m_Callable(callable) { } + ~FunctorWrapper() { + GILock lock; + m_Callable = bpy::object(); + } + RET operator()(const PAR1 ¶m1, const PAR2 ¶m2) { GILock lock; return (RET) m_Callable(param1, param2); @@ -1354,6 +1369,8 @@ bool PythonRunner::initPython(const QString &pythonPath) Py_InitializeEx(0); if (!Py_IsInitialized()) { + if (PyGILState_Check()) + PyEval_SaveThread(); return false; } @@ -1369,6 +1386,7 @@ bool PythonRunner::initPython(const QString &pythonPath) "sys.excepthook = lambda x, y, z: sys.__excepthook__(x, y, z)\n", mainNamespace); + PyEval_SaveThread(); return true; } catch (const bpy::error_already_set&) { qDebug("failed to init python"); @@ -1378,6 +1396,8 @@ bool PythonRunner::initPython(const QString &pythonPath) } else { qCritical("An unexpected C++ exception was thrown in python code"); } + if (PyGILState_Check()) + PyEval_SaveThread(); return false; } } From 5825661a4fd8d45363dfd9d6681a77f70b440a22 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Thu, 16 Apr 2020 00:53:29 +0100 Subject: [PATCH 002/341] Only initialise Python once When switching instances, the Python runner gets another initPython call, despite the previous interpreter existing. Most of the function was either a no-op or was redundant, and the GIL state got confusing. This is the simplest fix. If things catch fire that I haven't anticipated, instead of returning true, we can destroy the pre-existing interpreter and create a new one by running the rest of the function. --- src/runner/pythonrunner.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 9cc65d19..113e26c3 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -1347,6 +1347,8 @@ BOOST_PYTHON_MODULE(moprivate) bool PythonRunner::initPython(const QString &pythonPath) { + if (Py_IsInitialized()) + return true; try { if (!pythonPath.isEmpty() && !QFile::exists(pythonPath + "/python.exe")) { return false; From c5d4857f47f3a6389b9efe0080cb65f0826dad37 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Thu, 16 Apr 2020 00:56:57 +0100 Subject: [PATCH 003/341] Make functor converter variadic Duplicating code three times is silly. --- src/runner/pythonrunner.cpp | 121 +++++------------------------------- 1 file changed, 15 insertions(+), 106 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 113e26c3..8fdd89cd 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -724,100 +724,9 @@ int getArgCount(PyObject *object) { return result; } -template -struct Functor0_converter -{ - - struct FunctorWrapper - { - FunctorWrapper(boost::python::object callable) : m_Callable(callable) { - } - - ~FunctorWrapper() { - GILock lock; - m_Callable = bpy::object(); - } - - RET operator()() { - GILock lock; - return (RET) m_Callable(); - } - - boost::python::object m_Callable; - }; - - Functor0_converter() - { - bpy::converter::registry::push_back(&convertible, &construct, bpy::type_id>()); - } - - static void *convertible(PyObject *object) - { - if (!PyCallable_Check(object) - || (getArgCount(object) != 0)) { - return nullptr; - } - return object; - } - - static void construct(PyObject *object, bpy::converter::rvalue_from_python_stage1_data *data) - { - bpy::object callable(bpy::handle<>(bpy::borrowed(object))); - void *storage = ((bpy::converter::rvalue_from_python_storage>*)data)->storage.bytes; - new (storage) std::function(FunctorWrapper(callable)); - data->convertible = storage; - } -}; - - -template -struct Functor1_converter -{ - - struct FunctorWrapper - { - FunctorWrapper(boost::python::object callable) : m_Callable(callable) { - } - - ~FunctorWrapper() { - GILock lock; - m_Callable = bpy::object(); - } - - RET operator()(const PAR1 ¶m1) { - GILock lock; - return (RET) m_Callable(param1); - } - - boost::python::object m_Callable; - }; - - Functor1_converter() - { - bpy::converter::registry::push_back(&convertible, &construct, bpy::type_id>()); - } - - static void *convertible(PyObject *object) - { - if (!PyCallable_Check(object) - || (getArgCount(object) != 1)) { - return nullptr; - } - return object; - } - - static void construct(PyObject *object, bpy::converter::rvalue_from_python_stage1_data *data) - { - bpy::object callable(bpy::handle<>(bpy::borrowed(object))); - void *storage = ((bpy::converter::rvalue_from_python_storage>*)data)->storage.bytes; - new (storage) std::function(FunctorWrapper(callable)); - data->convertible = storage; - } -}; - -template -struct Functor2_converter +template +struct Functor_converter { struct FunctorWrapper @@ -830,23 +739,23 @@ struct Functor2_converter m_Callable = bpy::object(); } - RET operator()(const PAR1 ¶m1, const PAR2 ¶m2) { + RET operator()(const PARAMS &...params) { GILock lock; - return (RET) m_Callable(param1, param2); + return (RET) m_Callable(params...); } boost::python::object m_Callable; }; - Functor2_converter() + Functor_converter() { - bpy::converter::registry::push_back(&convertible, &construct, bpy::type_id>()); + bpy::converter::registry::push_back(&convertible, &construct, bpy::type_id>()); } static void *convertible(PyObject *object) { if (!PyCallable_Check(object) - || (getArgCount(object) != 2)) { + || (getArgCount(object) != sizeof...(PARAMS))) { return nullptr; } return object; @@ -855,8 +764,8 @@ struct Functor2_converter static void construct(PyObject *object, bpy::converter::rvalue_from_python_stage1_data *data) { bpy::object callable(bpy::handle<>(bpy::borrowed(object))); - void *storage = ((bpy::converter::rvalue_from_python_storage>*)data)->storage.bytes; - new (storage) std::function(FunctorWrapper(callable)); + void *storage = ((bpy::converter::rvalue_from_python_storage>*)data)->storage.bytes; + new (storage) std::function(FunctorWrapper(callable)); data->convertible = storage; } }; @@ -963,10 +872,10 @@ BOOST_PYTHON_MODULE(mobase) // TODO: ISaveGameInfoWidget bindings - Functor1_converter(); - Functor1_converter(); - Functor1_converter(); - Functor2_converter(); + Functor_converter(); + Functor_converter(); + Functor_converter(); + Functor_converter(); bpy::class_("IOrganizer") .def("createNexusBridge", bpy::pure_virtual(&IOrganizer::createNexusBridge), bpy::return_value_policy()) @@ -1108,7 +1017,7 @@ BOOST_PYTHON_MODULE(mobase) bpy::to_python_converter>(); QFlags_from_python_obj(); - Functor0_converter(); // converter for the onRefreshed-callback + Functor_converter(); // converter for the onRefreshed-callback bpy::enum_("PluginState") .value("missing", IPluginList::STATE_MISSING) @@ -1132,7 +1041,7 @@ BOOST_PYTHON_MODULE(mobase) bpy::to_python_converter>(); QFlags_from_python_obj(); - Functor2_converter(); // converter for the onModStateChanged-callback + Functor_converter(); // converter for the onModStateChanged-callback bpy::enum_("ModState") .value("exists", IModList::STATE_EXISTS) From 97a93a5989684f0bdb7fd57a60baae0e1956366f Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Sat, 25 Apr 2020 00:51:14 -0400 Subject: [PATCH 004/341] now using new cmakefiles --- CMakeLists.txt | 29 ++++--------- src/proxy/CMakeLists.txt | 74 +++------------------------------- src/runner/CMakeLists.txt | 85 ++++++--------------------------------- 3 files changed, 24 insertions(+), 164 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b2e4c5d0..2f715156 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,25 +1,10 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12) +cmake_minimum_required(VERSION 3.16) -ADD_COMPILE_OPTIONS($<$:/MP> $<$:$<$:/O2>> $<$:$<$:/O2>>) +project(plugin_python) +set(project_type plugin) +set(enable_warnings OFF) -SET(PROJ_NAME plugin_python) +include(../cmake_common/project.cmake) +add_subdirectory(src/proxy) -PROJECT(${PROJ_NAME}) - -SET(DEPENDENCIES_DIR CACHE PATH "") - -# hint to find qt in dependencies path -LIST(APPEND CMAKE_PREFIX_PATH ${QT_ROOT}/lib/cmake) - -# need to install python -IF(EXISTS "${PYTHON_ROOT}/PCbuild/python37.dll") - INSTALL(FILES ${PYTHON_ROOT}/PCbuild/python37.dll DESTINATION bin) - INSTALL(FILES ${PYTHON_ROOT}/PCbuild/python37.pdb DESTINATION pdb) -ELSEIF(EXISTS "${PYTHON_ROOT}/PCbuild/amd64/python27.dll") - INSTALL(FILES ${PYTHON_ROOT}/PCbuild/amd64/python37.dll DESTINATION bin) - INSTALL(FILES ${PYTHON_ROOT}/PCbuild/amd64/python37.pdb DESTINATION pdb) -ENDIF() - - -ADD_SUBDIRECTORY(src/runner) -ADD_SUBDIRECTORY(src/proxy) +add_subdirectory(src/runner) diff --git a/src/proxy/CMakeLists.txt b/src/proxy/CMakeLists.txt index ff53fef6..7ee8d361 100644 --- a/src/proxy/CMakeLists.txt +++ b/src/proxy/CMakeLists.txt @@ -1,70 +1,6 @@ -CMAKE_MINIMUM_REQUIRED (VERSION 2.8.11) +cmake_minimum_required(VERSION 3.16) +include(../../../cmake_common/src.cmake) -CMAKE_POLICY(SET CMP0020 NEW) -CMAKE_POLICY(SET CMP0043 NEW) - -SET(${PROJ_NAME}_SRCS proxypython.cpp) - -SET(${PROJ_NAME}_HDRS - proxypython.h - resource.h) - -SET(CMAKE_INCLUDE_CURRENT_DIR ON) -SET(CMAKE_AUTOMOC ON) -SET(CMAKE_AUTOUIC ON) -FIND_PACKAGE(Qt5Widgets REQUIRED) -FIND_PACKAGE(Qt5Network REQUIRED) -#QT5_WRAP_UI(${PROJ_NAME}_UIHDRS ${${PROJ_NAME}_FORMS}) - -SET(Boost_USE_STATIC_LIBS ON) -SET(Boost_USE_MULTITHREADED ON) -SET(Boost_USE_STATIC_RUNTIME OFF) -FIND_PACKAGE(Boost) - -IF (Boost_FOUND) - INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS}) -ENDIF (Boost_FOUND) - -SET(default_project_path "${DEPENDENCIES_DIR}/modorganizer_super") -GET_FILENAME_COMPONENT(${default_project_path} ${default_project_path} REALPATH) - -SET(project_path "${default_project_path}" CACHE PATH "path to the other mo projects") -SET(lib_path "${project_path}/../../install/libs") - -ADD_DEFINITIONS(-DUNICODE -D_UNICODE) - -INCLUDE_DIRECTORIES(${project_path}/uibase/src - ${CMAKE_SOURCE_DIR}/src/runner/ - ${PYTHON_ROOT}/Include) -LINK_DIRECTORIES(${lib_path}) - - -ADD_LIBRARY(${PROJ_NAME} SHARED ${${PROJ_NAME}_HDRS} ${${PROJ_NAME}_SRCS} ${${PROJ_NAME}_UIHDRS}) -TARGET_LINK_LIBRARIES(${PROJ_NAME} - Qt5::Widgets - Qt5::Network - ${Boost_LIBRARIES} - uibase) - -IF (MSVC) - SET_TARGET_PROPERTIES(${PROJ_NAME} PROPERTIES COMPILE_FLAGS "/std:c++latest") -ENDIF() -IF (MSVC AND CMAKE_SIZEOF_VOID_P EQUAL 4) - # 32 bits - SET_TARGET_PROPERTIES(${PROJ_NAME} PROPERTIES LINK_FLAGS "/LARGEADDRESSAWARE") -ENDIF() - -IF (NOT "${OPTIMIZE_COMPILE_FLAGS}" STREQUAL "") - SET_TARGET_PROPERTIES(${PROJ_NAME} PROPERTIES COMPILE_FLAGS_RELWITHDEBINFO ${OPTIMIZE_COMPILE_FLAGS}) -ENDIF() -IF (NOT "${OPTIMIZE_LINK_FLAGS}" STREQUAL "") - SET_TARGET_PROPERTIES(${PROJ_NAME} PROPERTIES LINK_FLAGS_RELWITHDEBINFO ${OPTIMIZE_LINK_FLAGS}) -ENDIF() - -############### -## Installation - -INSTALL(TARGETS ${PROJ_NAME} - RUNTIME DESTINATION bin/plugins) -INSTALL(FILES $ - DESTINATION pdb) +requires_library(python) +target_include_directories(${PROJECT_NAME} PRIVATE + ${CMAKE_SOURCE_DIR}/src/runner) diff --git a/src/runner/CMakeLists.txt b/src/runner/CMakeLists.txt index e237e074..2b7fbf56 100644 --- a/src/runner/CMakeLists.txt +++ b/src/runner/CMakeLists.txt @@ -1,77 +1,16 @@ -CMAKE_MINIMUM_REQUIRED (VERSION 2.8.11) +cmake_minimum_required(VERSION 3.16) -SET(PROJ_NAME pythonrunner) +project(pythonrunner) +set(project_type plugin) +set(enable_warnings OFF) +set(install_dir bin/plugins/data) -PROJECT(${PROJ_NAME}) +add_compile_definitions( + QT_NO_KEYWORDS + PYTHONRUNNER_LIBRARY) -CMAKE_POLICY(SET CMP0020 NEW) +include(../../../cmake_common/project.cmake) +include(../../../cmake_common/src.cmake) - -FILE(GLOB ${PROJ_NAME}_SRCS *.cpp) -FILE(GLOB ${PROJ_NAME}_HDRS *.h) - -SET(CMAKE_INCLUDE_CURRENT_DIR ON) -SET(CMAKE_AUTOMOC ON) -SET(CMAKE_AUTOUIC ON) -ADD_DEFINITIONS(-DQT_NO_KEYWORDS) -FIND_PACKAGE(Qt5Widgets REQUIRED) -FIND_PACKAGE(Qt5Network REQUIRED) -#QT5_WRAP_UI(${PROJ_NAME}_UIHDRS ${${PROJ_NAME}_FORMS}) -FIND_PACKAGE(Qt5LinguistTools) -QT5_CREATE_TRANSLATION(${PROJ_NAME}_translations_qm ${CMAKE_SOURCE_DIR}/src ${CMAKE_SOURCE_DIR}/src/${PROJ_NAME}_en.ts) - -SET(Boost_USE_STATIC_LIBS OFF) -SET(Boost_USE_MULTITHREADED ON) -SET(Boost_USE_STATIC_RUNTIME OFF) -FIND_PACKAGE(Boost REQUIRED) - -IF (Boost_FOUND) - INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS}) -ENDIF (Boost_FOUND) - -SET(default_project_path "${DEPENDENCIES_DIR}/modorganizer_super") -GET_FILENAME_COMPONENT(${default_project_path} ${default_project_path} REALPATH) - -SET(project_path "${default_project_path}" CACHE PATH "path to the other mo projects") -SET(lib_path "${project_path}/../../install/libs") - -INCLUDE_DIRECTORIES(${project_path}/uibase/src - ${project_path}/game_features/src - ${PYTHON_ROOT}/Include - ${PYTHON_ROOT}/PC) - -LINK_DIRECTORIES(${lib_path} - ${PYTHON_ROOT}/PCbuild - ${Boost_LIBRARY_DIRS}) - -ADD_LIBRARY(${PROJ_NAME} SHARED ${${PROJ_NAME}_HDRS} ${${PROJ_NAME}_SRCS} ${${PROJ_NAME}_translations_qm}) -TARGET_LINK_LIBRARIES(${PROJ_NAME} - Qt5::Widgets - Qt5::Network - uibase) - -ADD_DEFINITIONS(-DPYTHONRUNNER_LIBRARY) - -IF (MSVC) - SET_TARGET_PROPERTIES(${PROJ_NAME} PROPERTIES COMPILE_FLAGS "/std:c++latest") -ENDIF() -IF (MSVC AND CMAKE_SIZEOF_VOID_P EQUAL 4) - # 32 bits - SET_TARGET_PROPERTIES(${PROJ_NAME} PROPERTIES LINK_FLAGS "/LARGEADDRESSAWARE") -ENDIF() - -IF (NOT "${OPTIMIZE_COMPILE_FLAGS}" STREQUAL "") - SET_TARGET_PROPERTIES(${PROJ_NAME} PROPERTIES COMPILE_FLAGS_RELWITHDEBINFO ${OPTIMIZE_COMPILE_FLAGS}) -ENDIF() -IF (NOT "${OPTIMIZE_LINK_FLAGS}" STREQUAL "") - SET_TARGET_PROPERTIES(${PROJ_NAME} PROPERTIES LINK_FLAGS_RELWITHDEBINFO ${OPTIMIZE_LINK_FLAGS}) -ENDIF() - -############### -## Installation - -INSTALL(TARGETS ${PROJ_NAME} - RUNTIME DESTINATION bin/plugins/data - ARCHIVE DESTINATION libs) -INSTALL(FILES $ - DESTINATION pdb) +requires_project(game_features) +requires_library(python) From 332b8239bbccc6339f973bcf32f5397efbf84b4d Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Sat, 25 Apr 2020 00:51:20 -0400 Subject: [PATCH 005/341] unused --- src/proxy/embedrunner.rc | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 src/proxy/embedrunner.rc diff --git a/src/proxy/embedrunner.rc b/src/proxy/embedrunner.rc deleted file mode 100644 index dfdf8ef3..00000000 --- a/src/proxy/embedrunner.rc +++ /dev/null @@ -1,7 +0,0 @@ -#include "resource.h" - -#ifdef SCONS_BUILD -IDR_LOADER_DLL BINARY MOVEABLE PURE pythonRunner.dll -#else -IDR_LOADER_DLL BINARY MOVEABLE PURE "..\\runner\\pythonRunner.dll" -#endif From 168b9d7b0dce58a9a24bef55d50852038f5de94f Mon Sep 17 00:00:00 2001 From: Silarn Date: Fri, 1 May 2020 04:59:02 -0500 Subject: [PATCH 006/341] Appveyor CMAKE fix --- CMakeLists.txt | 6 +++++- src/proxy/CMakeLists.txt | 8 +++++++- src/runner/CMakeLists.txt | 11 +++++++++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f715156..e7a4b337 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,11 @@ project(plugin_python) set(project_type plugin) set(enable_warnings OFF) -include(../cmake_common/project.cmake) +if(DEFINED DEPENDENCIES_DIR) + include(${DEPENDENCIES_DIR}/modorganizer_super/cmake_common/project.cmake) +else() + include(../cmake_common/project.cmake) +endif() add_subdirectory(src/proxy) add_subdirectory(src/runner) diff --git a/src/proxy/CMakeLists.txt b/src/proxy/CMakeLists.txt index 7ee8d361..610f7568 100644 --- a/src/proxy/CMakeLists.txt +++ b/src/proxy/CMakeLists.txt @@ -1,5 +1,11 @@ cmake_minimum_required(VERSION 3.16) -include(../../../cmake_common/src.cmake) +# appveyor does not build modorganizer in its standard location, so use +# DEPENDENCIES_DIR to find cmake_common +if(DEFINED DEPENDENCIES_DIR) + include(${DEPENDENCIES_DIR}/modorganizer_super/cmake_common/src.cmake) +else() + include(../../../cmake_common/src.cmake) +endif() requires_library(python) target_include_directories(${PROJECT_NAME} PRIVATE diff --git a/src/runner/CMakeLists.txt b/src/runner/CMakeLists.txt index 2b7fbf56..0e6b464a 100644 --- a/src/runner/CMakeLists.txt +++ b/src/runner/CMakeLists.txt @@ -9,8 +9,15 @@ add_compile_definitions( QT_NO_KEYWORDS PYTHONRUNNER_LIBRARY) -include(../../../cmake_common/project.cmake) -include(../../../cmake_common/src.cmake) +# appveyor does not build modorganizer in its standard location, so use +# DEPENDENCIES_DIR to find cmake_common +if(DEFINED DEPENDENCIES_DIR) + include(${DEPENDENCIES_DIR}/modorganizer_super/cmake_common/project.cmake) + include(${DEPENDENCIES_DIR}/modorganizer_super/cmake_common/src.cmake) +else() + include(../../../cmake_common/project.cmake) + include(../../../cmake_common/src.cmake) +endif() requires_project(game_features) requires_library(python) From f25602cc6930bde36237b222b8069d49dd4651b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Fri, 1 May 2020 21:30:31 +0200 Subject: [PATCH 007/341] Update the wrapper and the python interface after change to IModInterface. --- src/runner/pythonrunner.cpp | 4 ++-- src/runner/uibasewrappers.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 8fdd89cd..01fb6c15 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -622,7 +622,7 @@ struct QClass_converters wrapper = reinterpret_cast(objPtr); return wrapper->super.data; } else { - if (std::is_same_v) + if constexpr (std::is_same_v) { // QStringLists aren't wrapped by PyQt - regular Python string/unicode lists are used instead bpy::extract> extractor(objPtr); @@ -994,7 +994,7 @@ BOOST_PYTHON_MODULE(mobase) .def("addCategory", bpy::pure_virtual(&IModInterface::addCategory)) .def("removeCategory", bpy::pure_virtual(&IModInterface::removeCategory)) .def("categories", bpy::pure_virtual(&IModInterface::categories)) - .def("setGameName", bpy::pure_virtual(&IModInterface::setGameName)) + .def("setGamePlugin", bpy::pure_virtual(&IModInterface::setGamePlugin)) .def("setName", bpy::pure_virtual(&IModInterface::setName)) .def("remove", bpy::pure_virtual(&IModInterface::remove)) ; diff --git a/src/runner/uibasewrappers.h b/src/runner/uibasewrappers.h index b31bb608..d37f422b 100644 --- a/src/runner/uibasewrappers.h +++ b/src/runner/uibasewrappers.h @@ -443,12 +443,12 @@ struct IModInterfaceWrapper: MOBase::IModInterface, boost::python::wrapperget_override("setNexusID")(nexusID); } virtual void setInstallationFile(const QString &fileName) override { this->get_override("setInstallationFile")(fileName); } virtual void addNexusCategory(int categoryID) override { this->get_override("addNexusCategory")(categoryID); } - virtual void setGameName(const QString &gameName) override { this->get_override("setGameName")(gameName); } + virtual void setGamePlugin(const MOBase::IPluginGame *gamePlugin) override { this->get_override("setGamePlugin")(gamePlugin); } virtual bool setName(const QString &name) override { return this->get_override("setName")(name); } virtual bool remove() override { return this->get_override("remove")(); } virtual void addCategory(const QString &categoryName) override { this->get_override("addCategory")(categoryName); } virtual bool removeCategory(const QString &categoryName) override { return this->get_override("removeCategory")(categoryName); } - virtual QStringList categories() override { return this->get_override("categories")(); } + virtual QStringList categories() const override { return this->get_override("categories")(); } }; From d82640455faf17e20ccdbada1409efb167b0dbf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Fri, 1 May 2020 21:32:08 +0200 Subject: [PATCH 008/341] Update wrappers and expose the new file tree interface. --- src/runner/pythonrunner.cpp | 161 ++++++++++++++++++++++++++++++++++-- src/runner/uibasewrappers.h | 10 ++- 2 files changed, 162 insertions(+), 9 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 01fb6c15..0987b7d1 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -3,6 +3,7 @@ #pragma warning( disable : 4100 ) #pragma warning( disable : 4996 ) +#include #include #include #include @@ -781,6 +782,44 @@ static PyObject *waitForApplication(const bpy::object &self, size_t handle) } +/** + * @brief Call policy that automatically downcast shared pointer of type FromType + * to shared pointer of type ToType. + */ +template +struct DowncastConverter { + + bool convertible() const { return true; } + + inline PyObject* operator()(std::shared_ptr p) const { + if (p == nullptr) { + return bpy::detail::none(); + } + else { + auto downcast_p = std::dynamic_pointer_cast(p); + bpy::object p_value = downcast_p == nullptr ? bpy::object{ p } : bpy::object{ downcast_p }; + return bpy::incref(p_value.ptr()); + } + } + + inline PyTypeObject const* get_pytype() const { + return bpy::converter::registered_pytype::get_pytype(); + } + +}; + +template +struct DowncastReturn { + + template + struct apply { + static_assert(std::is_convertible_v>); + using type = DowncastConverter; + }; + +}; + + BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS(updateWithQuality, MOBase::GuessedValue::update, 2, 2) @@ -918,6 +957,118 @@ BOOST_PYTHON_MODULE(mobase) .def("modsSortedByProfilePriority", bpy::pure_virtual(&IOrganizer::modsSortedByProfilePriority)) ; + // FileTreeEntry and IFileTree are only managed by shared ptr. + bpy::register_ptr_to_python>(); + bpy::register_ptr_to_python>(); + + // FileTreeEntry Scope: + auto fileTreeEntryClass = bpy::class_, boost::noncopyable>("FileTreeEntry", bpy::no_init); + { + + bpy::scope scope = fileTreeEntryClass; + + + bpy::enum_("FileType") + .value("FILE_OR_DIRECTORY", FileTreeEntry::FILE_OR_DIRECTORY) + .value("FILE", FileTreeEntry::FILE) + .value("DIRECTORY", FileTreeEntry::DIRECTORY) + .export_values() + ; + + fileTreeEntryClass + + .def("isFile", &FileTreeEntry::isFile) + .def("isDir", &FileTreeEntry::isDir) + .def("getFileType", &FileTreeEntry::fileType) + // This should probably not be exposed in python since we provide automatic downcast: + // .def("getTree", static_cast(FileTreeEntry::*)()>(&FileTreeEntry::astree)) + .def("getName", &FileTreeEntry::name) + .def("getSuffix", &FileTreeEntry::suffix) + .def("getTime", &FileTreeEntry::time) + .def("getParent", static_cast(FileTreeEntry::*)()>(&FileTreeEntry::parent)) + .def("getPath", &FileTreeEntry::path, bpy::arg("sep") = "\\") + .def("getPathFrom", &FileTreeEntry::pathFrom, bpy::arg("sep") = "\\") + + // Mutable operation: + .def("setTime", &FileTreeEntry::setTime) + .def("detach", &FileTreeEntry::detach) + .def("moveTo", &FileTreeEntry::moveTo) + + // Special methods for debug: + .def("__str__", &FileTreeEntry::name) + .def("__repr__", +[](const FileTreeEntry* entry) { return "FileTreeEntry(" + entry->name() + ")"; }) + ; + } + + // IFileTree scope: + auto iFileTreeClass = bpy::class_, boost::noncopyable>("IFileTree", bpy::no_init);; + { + + bpy::scope scope = iFileTreeClass; + + bpy::enum_("InsertPolicy") + .value("FAIL_IF_EXISTS", IFileTree::InsertPolicy::FAIL_IF_EXISTS) + .value("REPLACE", IFileTree::InsertPolicy::REPLACE) + .value("MERGE", IFileTree::InsertPolicy::MERGE) + .export_values() + ; + + iFileTreeClass + + // Non-mutable operations (note: iterator and some methods are at the end with + // special python methods): + .def("exists", static_cast(&IFileTree::exists), (bpy::arg("type") = IFileTree::FILE_OR_DIRECTORY)) + .def("find", static_cast(IFileTree::*)(QString, IFileTree::FileTypes)>(&IFileTree::find), + bpy::arg("type") = IFileTree::FILE_OR_DIRECTORY, bpy::return_value_policy>()) + .def("getPathTo", &IFileTree::pathTo, bpy::arg("sep") = "\\") + + // Kind-of-static operations: + .def("createOrphanTree", &IFileTree::createOrphanTree, bpy::arg("name") = "") + + // Mutable operations: + .def("addFile", &IFileTree::addFile, bpy::arg("time") = QDateTime()) + .def("addDirectory", &IFileTree::addDirectory) + .def("insert", +[]( + IFileTree* p, std::shared_ptr entry, IFileTree::InsertPolicy insertPolicy) { + return p->insert(entry, insertPolicy) != p->end(); }, bpy::arg("policy") = IFileTree::InsertPolicy::FAIL_IF_EXISTS) + + .def("merge", +[](IFileTree* p, std::shared_ptr other, bool returnOverwrites) -> bpy::object { + IFileTree::OverwritesType overwrites; + auto result = p->merge(other, returnOverwrites ? &overwrites : nullptr); + if (result == IFileTree::MERGE_FAILED) { + return bpy::object{ false }; + } + if (returnOverwrites) { + return bpy::object{ overwrites }; + } + return bpy::object{ result }; + }, bpy::arg("overwrites") = false) + + .def("move", &IFileTree::move, bpy::arg("policy") = IFileTree::InsertPolicy::FAIL_IF_EXISTS) + + .def("remove", +[](IFileTree* p, QString name) { return p->erase(name).first != p->end(); }) + .def("remove", +[](IFileTree* p, std::shared_ptr entry) { return p->erase(entry) != p->end(); }) + + .def("clear", &IFileTree::clear) + .def("removeAll", &IFileTree::removeAll) + .def("removeIf", +[](IFileTree* p, boost::python::object fn) { + return p->removeIf(fn); + }) + + // Special methods: + .def("__getitem__", static_cast(IFileTree::*)(std::size_t)>(&IFileTree::at), + bpy::return_value_policy>()) + .def("__iter__", bpy::range>>( + static_cast(&IFileTree::begin), + static_cast(&IFileTree::end))) + .def("__len__", &IFileTree::size) + .def("__bool__", +[](const IFileTree* tree) { return !tree->empty(); }) + .def("__str__", &FileTreeEntry::name) + .def("__repr__", +[](const IFileTree* entry) { return "IFileTree(" + entry->name() + ")"; }) + ; + } + + bpy::class_("IProfile") .def("name", bpy::pure_virtual(&IProfile::name)) .def("absolutePath", bpy::pure_virtual(&IProfile::absolutePath)) @@ -977,11 +1128,11 @@ BOOST_PYTHON_MODULE(mobase) ; bpy::class_("IInstallationManager") - .def("extractFile", bpy::pure_virtual(&IInstallationManager::extractFile)) - .def("extractFiles", bpy::pure_virtual(&IInstallationManager::extractFiles)) - .def("installArchive", bpy::pure_virtual(&IInstallationManager::installArchive)) - .def("setURL", bpy::pure_virtual(&IInstallationManager::setURL)) - ; + .def("extractFile", bpy::pure_virtual(&IInstallationManager::extractFile)) + .def("extractFiles", bpy::pure_virtual(&IInstallationManager::extractFiles)) + .def("installArchive", bpy::pure_virtual(&IInstallationManager::installArchive)) + .def("setURL", bpy::pure_virtual(&IInstallationManager::setURL)) + ; bpy::class_("IModInterface") .def("name", bpy::pure_virtual(&IModInterface::name)) diff --git a/src/runner/uibasewrappers.h b/src/runner/uibasewrappers.h index d37f422b..bab65bb2 100644 --- a/src/runner/uibasewrappers.h +++ b/src/runner/uibasewrappers.h @@ -23,6 +23,7 @@ #include #include #include +#include "ifiletree.h" #include "error.h" #include "gilock.h" @@ -427,10 +428,11 @@ struct IModRepositoryBridgeWrapper: MOBase::IModRepositoryBridge, boost::python: struct IInstallationManagerWrapper: MOBase::IInstallationManager, boost::python::wrapper { - virtual QString extractFile(const QString &fileName) { return this->get_override("extractFile")(fileName); } - virtual QStringList extractFiles(const QStringList &files, bool flatten) { return this->get_override("extractFiles")(files, flatten); } - virtual MOBase::IPluginInstaller::EInstallResult installArchive(MOBase::GuessedValue &modName, const QString &archiveFile, const int &modId = 0) { return this->get_override("installArchive")(modName, archiveFile, modId); } - virtual void setURL(QString const &url) { this->get_override("setURL")(url); } + virtual QString extractFile(std::shared_ptr entry) override { return this->get_override("extractFile")(entry); } + virtual QStringList extractFiles(std::vector> const& entries) override { return this->get_override("extractFiles")(entries); } + virtual MOBase::IPluginInstaller::EInstallResult installArchive(MOBase::GuessedValue &modName, const QString &archiveFile, int modId = 0) override { + return this->get_override("installArchive")(modName, archiveFile, modId); } + virtual void setURL(QString const &url) override { this->get_override("setURL")(url); } }; struct IModInterfaceWrapper: MOBase::IModInterface, boost::python::wrapper From 6cececc7a057552c788b21081a2124d0c02558d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Fri, 1 May 2020 21:32:30 +0200 Subject: [PATCH 009/341] Fix issue with default QDateTime() throwing error. --- src/runner/pythonrunner.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 0987b7d1..69a95a2b 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -826,6 +826,9 @@ BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS(updateWithQuality, MOBase::GuessedValue(); QVariant_from_python_obj(); From c9115e7ad0869b4b253274a938a1cfbd2c7a9c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sat, 2 May 2020 00:24:45 +0200 Subject: [PATCH 010/341] Add python interface for IPluginInstallerSimple. --- src/runner/proxypluginwrappers.cpp | 72 +++++--- src/runner/proxypluginwrappers.h | 38 ++++- src/runner/pythonrunner.cpp | 100 +++++------- src/runner/tuple_helper.h | 253 +++++++++++++++++++++++++++++ 4 files changed, 371 insertions(+), 92 deletions(-) create mode 100644 src/runner/tuple_helper.h diff --git a/src/runner/proxypluginwrappers.cpp b/src/runner/proxypluginwrappers.cpp index 127835e3..fe87be8a 100644 --- a/src/runner/proxypluginwrappers.cpp +++ b/src/runner/proxypluginwrappers.cpp @@ -3,9 +3,12 @@ #include "gilock.h" #include #include + #include "pythonwrapperutilities.h" #include "sipApiAccess.h" +#include + namespace boost { // See bug https://connect.microsoft.com/VisualStudio/Feedback/Details/2852624 @@ -286,27 +289,55 @@ std::map IPluginGameWrapper::featureList() const } /// end IPluginGame Wrapper ///////////////////////////////////// -/// IPluginInstallerCustom Wrapper +/// IPluginInstaller macro +#define COMMON_I_PLUGIN_INSTALLER_WRAPPER_DEFINITIONS(class_name) \ +unsigned int class_name::priority() const { return basicWrapperFunctionImplementation(this, "priority"); } \ +bool class_name::isManualInstaller() const { return basicWrapperFunctionImplementation(this, "isManualInstaller"); } \ +bool class_name::isArchiveSupported(std::shared_ptr tree) const { return basicWrapperFunctionImplementation(this, "isArchiveSupported", tree); } +/// end IPluginInstaller macro +///////////////////////////////////// +/// IPluginInstaller Wrapper + +COMMON_I_PLUGIN_WRAPPER_DEFINITIONS(IPluginInstallerSimpleWrapper) +COMMON_I_PLUGIN_INSTALLER_WRAPPER_DEFINITIONS(IPluginInstallerSimpleWrapper) + +IPluginInstaller::EInstallResult IPluginInstallerSimpleWrapper::install( + GuessedValue& modName, std::shared_ptr& tree, + QString& version, int& nexusID) +{ + namespace bpy = boost::python; + + using return_type = boost::variant, boost::tuple, QString, int>> ; + auto ret = basicWrapperFunctionImplementation(this, "install", boost::ref(modName), tree, version, nexusID); + + return boost::apply_visitor([&](auto const& t) { + using type = std::decay_t; + if constexpr (std::is_same_v) { + return t; + } + else if constexpr (std::is_same_v>) { + tree = t; + return IPluginInstaller::RESULT_SUCCESS; + } + else if constexpr (std::is_same_v, QString, int>>) { + tree = boost::get<0>(t); + version = boost::get<1>(t); + nexusID = boost::get<2>(t); + return IPluginInstaller::RESULT_SUCCESS; + } + else { + static_assert("Type not handled in boost::apply_visitor."); + } + }, ret); +} + +/// end IPluginInstallerSimple Wrapper +///////////////////////////////////// +/// IPluginInstallerCustom Wrapper COMMON_I_PLUGIN_WRAPPER_DEFINITIONS(IPluginInstallerCustomWrapper) - -unsigned int IPluginInstallerCustomWrapper::priority() const -{ - return basicWrapperFunctionImplementation(this, "priority"); -} - -bool IPluginInstallerCustomWrapper::isManualInstaller() const -{ - return basicWrapperFunctionImplementation(this, "isManualInstaller"); -} - -bool IPluginInstallerCustomWrapper::isArchiveSupported(const DirectoryTree &archiveTree) const -{ - //return basicWrapperFunctionImplementation(this, "isArchiveSupported", archiveTree); - // This was a stub implementation when I got here, and a real one won't compile. - return false; -} +COMMON_I_PLUGIN_INSTALLER_WRAPPER_DEFINITIONS(IPluginInstallerCustomWrapper) bool IPluginInstallerCustomWrapper::isArchiveSupported(const QString &archiveName) const { @@ -324,11 +355,6 @@ IPluginInstaller::EInstallResult IPluginInstallerCustomWrapper::install(GuessedV return basicWrapperFunctionImplementation(this, "install", modName, gameName, archiveName, version, modID); } - -void IPluginInstallerCustomWrapper::setParentWidget(QWidget *parent) -{ - basicWrapperFunctionImplementation(this, "setParentWidget", parent); -} /// end IPluginInstallerCustom Wrapper ///////////////////////////// /// IPluginModPage Wrapper diff --git a/src/runner/proxypluginwrappers.h b/src/runner/proxypluginwrappers.h index 712707a5..c24d4ad6 100644 --- a/src/runner/proxypluginwrappers.h +++ b/src/runner/proxypluginwrappers.h @@ -132,25 +132,47 @@ class IPluginGameWrapper : public MOBase::IPluginGame, public boost::python::wra }; +#define COMMON_I_PLUGIN_INSTALLER_WRAPPER_DECLARATIONS public: \ +using IPluginInstaller::parentWidget; \ +using IPluginInstaller::manager; \ +virtual unsigned int priority() const override; \ +virtual bool isManualInstaller() const override; \ +virtual bool isArchiveSupported(std::shared_ptr tree) const override; + + +class IPluginInstallerSimpleWrapper : public MOBase::IPluginInstallerSimple, public boost::python::wrapper +{ + Q_OBJECT + Q_INTERFACES(MOBase::IPlugin MOBase::IPluginInstaller MOBase::IPluginInstallerSimple) + + COMMON_I_PLUGIN_WRAPPER_DECLARATIONS + COMMON_I_PLUGIN_INSTALLER_WRAPPER_DECLARATIONS + +public: + static constexpr const char* className = "IPluginInstallerSimpleWrapper"; + using boost::python::wrapper::get_override; + + virtual EInstallResult install(MOBase::GuessedValue& modName, std::shared_ptr& tree, + QString& version, int& nexusID) override; +}; + class IPluginInstallerCustomWrapper : public MOBase::IPluginInstallerCustom, public boost::python::wrapper { Q_OBJECT Q_INTERFACES(MOBase::IPlugin MOBase::IPluginInstaller MOBase::IPluginInstallerCustom) COMMON_I_PLUGIN_WRAPPER_DECLARATIONS + + COMMON_I_PLUGIN_INSTALLER_WRAPPER_DECLARATIONS + public: static constexpr const char* className = "IPluginInstallerCustomWrapper"; using boost::python::wrapper::get_override; - virtual unsigned int priority() const; - virtual bool isManualInstaller() const; - virtual bool isArchiveSupported(const MOBase::DirectoryTree &tree) const; - virtual bool isArchiveSupported(const QString &archiveName) const; - virtual std::set supportedExtensions() const; + virtual bool isArchiveSupported(const QString &archiveName) const override; + virtual std::set supportedExtensions() const override; virtual EInstallResult install(MOBase::GuessedValue &modName, QString gameName, const QString &archiveName, - const QString &version, int modID); - virtual void setParentWidget(QWidget *parent); - + const QString &version, int modID) override; }; diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 69a95a2b..bacbdaa7 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -25,9 +25,10 @@ #ifndef Q_MOC_RUN #include +#include +#include "tuple_helper.h" #endif - MOBase::IOrganizer *s_Organizer = nullptr; @@ -69,6 +70,7 @@ IPythonRunner *CreatePythonRunner(MOBase::IOrganizer *moInfo, const QString &pyt using namespace MOBase; namespace bpy = boost::python; +namespace mp11 = boost::mp11; struct QString_to_python_str { @@ -161,58 +163,6 @@ struct HANDLE_converters }; -template -struct GuessedValue_converters -{ - struct GuessedValue_to_python - { - static PyObject *convert(const GuessedValue &var) { - bpy::list result; - const std::set &values = var.variants(); - for (auto iter = values.begin(); iter != values.end(); ++iter) { - result.append(bpy::make_tuple(*iter, GUESS_GOOD)); - } - return bpy::incref(result.ptr()); - } - }; - - struct GuessedValue_from_python - { - GuessedValue_from_python() { - bpy::converter::registry::push_back(&convertible, &construct, bpy::type_id >()); - } - - static void *convertible(PyObject *objPtr) { - if PyList_Check(objPtr) { - return objPtr; - } else { - return nullptr; - } - } - - static void construct(PyObject *objPtr, bpy::converter::rvalue_from_python_stage1_data* data) { - void *storage = ((bpy::converter::rvalue_from_python_storage >*)data)->storage.bytes; - GuessedValue *result = new (storage) GuessedValue(); - - bpy::list source(bpy::handle<>(bpy::borrowed(objPtr))); - int length = bpy::len(source); - for (int i = 0; i < length; ++i) { - bpy::tuple cell = bpy::extract(source[i]); - result->update(bpy::extract(cell[0]), bpy::extract(cell[1])); - } - - data->convertible = storage; - } - }; - - GuessedValue_converters() - { - GuessedValue_from_python(); - bpy::to_python_converter, GuessedValue_to_python>(); - } -}; - - template struct QMap_converters { @@ -819,8 +769,19 @@ struct DowncastReturn { }; - -BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS(updateWithQuality, MOBase::GuessedValue::update, 2, 2) +/** + * @brief Register implicit conversion from any of the decayed type from the variant + * to the variant type. + */ +template +void register_implicit_variant() { + mp11::mp_for_each< + mp11::mp_list> ( + [](auto t) { + // take advantage of built-in conversions + bpy::implicitly_convertible, boost::variant>(); + }); +} BOOST_PYTHON_MODULE(mobase) @@ -962,7 +923,9 @@ BOOST_PYTHON_MODULE(mobase) // FileTreeEntry and IFileTree are only managed by shared ptr. bpy::register_ptr_to_python>(); + bpy::register_ptr_to_python>(); bpy::register_ptr_to_python>(); + bpy::register_ptr_to_python>(); // FileTreeEntry Scope: auto fileTreeEntryClass = bpy::class_, boost::noncopyable>("FileTreeEntry", bpy::no_init); @@ -1162,11 +1125,15 @@ BOOST_PYTHON_MODULE(mobase) .value("user", MOBase::GUESS_USER) ; - bpy::class_, boost::noncopyable>("GuessedString") + bpy::class_, boost::noncopyable>("GuessedString", bpy::no_init) .def("update", - static_cast &(GuessedValue::*)(const QString&, EGuessQuality)>(&GuessedValue::update), - bpy::return_value_policy(), updateWithQuality()) + static_cast& (GuessedValue::*)(const QString&)>(&GuessedValue::update), + bpy::return_self<>()) + .def("update", + static_cast& (GuessedValue::*)(const QString&, EGuessQuality)>(&GuessedValue::update), + bpy::return_self<>()) .def("variants", &MOBase::GuessedValue::variants, bpy::return_value_policy()) + .def("__str__", &MOBase::GuessedValue::operator const QString&, bpy::return_value_policy()) ; bpy::to_python_converter>(); @@ -1315,6 +1282,8 @@ BOOST_PYTHON_MODULE(mobase) .def("featureUnmanagedMods", &MOBase::IPluginGame::feature, bpy::return_value_policy()) ; + bpy::register_tuple, QString, int>>(); + register_implicit_variant, boost::tuple, QString, int>>(); bpy::enum_("InstallResult") .value("success", MOBase::IPluginInstaller::RESULT_SUCCESS) .value("failed", MOBase::IPluginInstaller::RESULT_FAILED) @@ -1323,8 +1292,18 @@ BOOST_PYTHON_MODULE(mobase) .value("notAttempted", MOBase::IPluginInstaller::RESULT_NOTATTEMPTED) ; + bpy::class_("IPluginInstallerSimple") + .def("install", &IPluginInstallerSimple::install) + .def("getParentWidget", &IPluginInstallerSimpleWrapper::parentWidget, bpy::return_value_policy()) + .def("getManager", &IPluginInstallerSimpleWrapper::manager, bpy::return_value_policy()) + ; + bpy::class_("IPluginInstallerCustom") - .def("setParentWidget", bpy::pure_virtual(&MOBase::IPluginInstallerCustom::setParentWidget)) + .def("isArchiveSupported", &IPluginInstallerCustom::isArchiveSupported) + .def("supportedExtensions", &IPluginInstallerCustom::supportedExtensions) + .def("install", &IPluginInstallerCustom::install) + .def("getParentWidget", &IPluginInstallerCustomWrapper::parentWidget, bpy::return_value_policy()) + .def("getManager", &IPluginInstallerCustomWrapper::manager, bpy::return_value_policy()) ; bpy::class_("IPluginModPage") @@ -1338,8 +1317,6 @@ BOOST_PYTHON_MODULE(mobase) .def("setParentWidget", bpy::pure_virtual(&MOBase::IPluginTool::setParentWidget)) ; - GuessedValue_converters(); - HANDLE_converters(); //bpy::to_python_converter(); @@ -1521,6 +1498,7 @@ QList PythonRunner::instantiate(const QString &pluginName) // Must try the wrapper because it's only a plugin extension interface in C++, so doesn't extend QObject TRY_PLUGIN_TYPE(IPluginFileMapperWrapper, pluginObj); TRY_PLUGIN_TYPE(IPluginInstallerCustom, pluginObj); + TRY_PLUGIN_TYPE(IPluginInstallerSimple, pluginObj); TRY_PLUGIN_TYPE(IPluginModPage, pluginObj); TRY_PLUGIN_TYPE(IPluginPreview, pluginObj); TRY_PLUGIN_TYPE(IPluginTool, pluginObj); diff --git a/src/runner/tuple_helper.h b/src/runner/tuple_helper.h new file mode 100644 index 00000000..10ed25bc --- /dev/null +++ b/src/runner/tuple_helper.h @@ -0,0 +1,253 @@ +// From: https://pyplusplus.readthedocs.io/en/latest/troubleshooting_guide/automatic_conversion/tuples.hpp.html +// Copyright 2004-2007 Roman Yakovenko. +// Distributed under the Boost Software License, Version 1.0. (See +// accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#ifndef TUPLES_HPP_16_JAN_2007 +#define TUPLES_HPP_16_JAN_2007 + +#include +#include +#include //len function +#include +#include + +/** + * Converts boost::tuples::tuple<...> to\from Python tuple + * + * The conversion is done "on-the-fly", you should only register the conversion + * with your tuple classes. + * For example: + * + * typedef boost::tuples::tuple< int, double, std::string > triplet; + * boost::python::register_tuple< triplet >(); + * + * That's all. After this point conversion to\from next types will be handled + * by Boost.Python library: + * + * triplet + * triplet& ( return type only ) + * const triplet + * const triplet& + * + * Implementation description. + * The conversion uses Boost.Python custom r-value converters. r-value converters + * is very powerful and undocumented feature of the library. The only documentation + * we have is http://boost.org/libs/python/doc/v2/faq.html#custom_string . + * + * The conversion consists from two parts: "to" and "from". + * + * "To" conversion + * The "to" part is pretty easy and well documented ( http://docs.python.org/api/api.html ). + * You should use Python C API to create an instance of a class and than you + * initialize the relevant members of the instance. + * + * "From" conversion + * Lets start from analyzing one of the use case Boost.Python library have to + * deal with: + * + * void do_smth( const triplet& arg ){...} + * + * In order to allow calling this function from Python, the library should keep + * parameter "arg" alive until the function returns. In other words, the library + * should provide instances life-time management. The provided interface is not + * ideal and could be improved. You have to implement two functions: + * + * void* convertible( PyObject* obj ) + * Checks whether the "obj" could be converted to an instance of the desired + * class. If true, the function should return "obj", otherwise NULL + * + * void construct( PyObject* obj, converter::rvalue_from_python_stage1_data* data) + * Constructs the instance of the desired class. This function will be called + * if and only if "convertible" function returned true. The first argument + * is Python object, which was passed as parameter to "convertible" function. + * The second object is some kind of memory allocator for one object. Basically + * it keeps a memory chunk. You will use the memory for object allocation. + * + * For some unclear for me reason, the library implements "C style Inheritance" + * ( http://www.embedded.com/97/fe29712.htm ). So, in order to create new + * object in the storage you have to cast to the "right" class: + * + * typedef converter::rvalue_from_python_storage storage_t; + * storage_t* the_storage = reinterpret_cast( data ); + * void* memory_chunk = the_storage->storage.bytes; + * + * "memory_chunk" points to the memory, where the instance will be allocated. + * + * In order to create object at specific location, you should use placement new + * operator: + * + * your_type_t* instance = new (memory_chunk) your_type_t(); + * + * Now, you can continue to initialize the instance. + * + * instance->set_xyz = read xyz from obj + * + * If "your_type_t" constructor requires some arguments, "read" the Python + * object before you call the constructor: + * + * xyz_type xyz = read xyz from obj + * your_type_t* instance = new (memory_chunk) your_type_t(xyz); + * + * Hint: + * In most case you don't really need\have to work with C Python API. Let + * Boost.Python library to do some work for you! + * + **/ + +namespace boost { + namespace python { + + namespace details { + + //Small helper function, introduced to allow short syntax for index incrementing + template< int index> + typename mpl::next< mpl::int_< index > >::type increment_index() { + typedef typename mpl::next< mpl::int_< index > >::type next_index_type; + return next_index_type(); + } + + } + + template< class TTuple > + struct to_py_tuple { + + typedef mpl::int_< tuples::length< TTuple >::value > length_type; + + static PyObject* convert(const TTuple& c_tuple) { + list values; + //add all c_tuple items to "values" list + convert_impl(c_tuple, values, mpl::int_< 0 >(), length_type()); + //create Python tuple from the list + return incref(python::tuple(values).ptr()); + } + + private: + + template< int index, int length > + static void + convert_impl(const TTuple& c_tuple, list& values, mpl::int_< index >, mpl::int_< length >) { + values.append(c_tuple.template get< index >()); + convert_impl(c_tuple, values, details::increment_index(), length_type()); + } + + template< int length > + static void + convert_impl(const TTuple&, list& values, mpl::int_< length >, mpl::int_< length >) + {} + + }; + + + template< class TTuple> + struct from_py_sequence { + + typedef TTuple tuple_type; + + typedef mpl::int_< tuples::length< TTuple >::value > length_type; + + static void* + convertible(PyObject* py_obj) { + + if (!PySequence_Check(py_obj)) { + return 0; + } + + if (!PyObject_HasAttrString(py_obj, "__len__")) { + return 0; + } + + python::object py_sequence(handle<>(borrowed(py_obj))); + + if (tuples::length< TTuple >::value != len(py_sequence)) { + return 0; + } + + if (convertible_impl(py_sequence, mpl::int_< 0 >(), length_type())) { + return py_obj; + } + else { + return 0; + } + } + + static void + construct(PyObject* py_obj, converter::rvalue_from_python_stage1_data* data) { + typedef converter::rvalue_from_python_storage storage_t; + storage_t* the_storage = reinterpret_cast(data); + void* memory_chunk = the_storage->storage.bytes; + TTuple* c_tuple = new (memory_chunk) TTuple(); + data->convertible = memory_chunk; + + python::object py_sequence(handle<>(borrowed(py_obj))); + construct_impl(py_sequence, *c_tuple, mpl::int_< 0 >(), length_type()); + } + + static TTuple to_c_tuple(PyObject* py_obj) { + if (!convertible(py_obj)) { + throw std::runtime_error("Unable to construct boost::tuples::tuple from Python object!"); + } + TTuple c_tuple; + python::object py_sequence(handle<>(borrowed(py_obj))); + construct_impl(py_sequence, c_tuple, mpl::int_< 0 >(), length_type()); + return c_tuple; + } + + private: + + template< int index, int length > + static bool + convertible_impl(const python::object& py_sequence, mpl::int_< index >, mpl::int_< length >) { + + typedef typename tuples::element< index, TTuple>::type element_type; + + object element = py_sequence[index]; + extract type_checker(element); + if (!type_checker.check()) { + return false; + } + else { + return convertible_impl(py_sequence, details::increment_index(), length_type()); + } + } + + template< int length > + static bool + convertible_impl(const python::object& py_sequence, mpl::int_< length >, mpl::int_< length >) { + return true; + } + + template< int index, int length > + static void + construct_impl(const python::object& py_sequence, TTuple& c_tuple, mpl::int_< index >, mpl::int_< length >) { + + typedef typename tuples::element< index, TTuple>::type element_type; + + object element = py_sequence[index]; + c_tuple.template get< index >() = extract(element); + + construct_impl(py_sequence, c_tuple, details::increment_index(), length_type()); + } + + template< int length > + static void + construct_impl(const python::object& py_sequence, TTuple& c_tuple, mpl::int_< length >, mpl::int_< length >) + {} + + }; + + template< class TTuple> + void register_tuple() { + + to_python_converter< TTuple, to_py_tuple >(); + + converter::registry::push_back(&from_py_sequence::convertible + , &from_py_sequence::construct + , type_id()); + }; + + } +} //boost::python + +#endif//TUPLES_HPP_16_JAN_2007 \ No newline at end of file From 626a7bc6705eab6968099c1fe6d345aa3a0f1b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sat, 2 May 2020 13:22:34 +0200 Subject: [PATCH 011/341] Update python bindings for tree to be consistent with others. --- src/runner/pythonrunner.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index bacbdaa7..7b42272f 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -945,15 +945,15 @@ BOOST_PYTHON_MODULE(mobase) .def("isFile", &FileTreeEntry::isFile) .def("isDir", &FileTreeEntry::isDir) - .def("getFileType", &FileTreeEntry::fileType) + .def("fileType", &FileTreeEntry::fileType) // This should probably not be exposed in python since we provide automatic downcast: // .def("getTree", static_cast(FileTreeEntry::*)()>(&FileTreeEntry::astree)) - .def("getName", &FileTreeEntry::name) - .def("getSuffix", &FileTreeEntry::suffix) - .def("getTime", &FileTreeEntry::time) - .def("getParent", static_cast(FileTreeEntry::*)()>(&FileTreeEntry::parent)) - .def("getPath", &FileTreeEntry::path, bpy::arg("sep") = "\\") - .def("getPathFrom", &FileTreeEntry::pathFrom, bpy::arg("sep") = "\\") + .def("name", &FileTreeEntry::name) + .def("suffix", &FileTreeEntry::suffix) + .def("time", &FileTreeEntry::time) + .def("parent", static_cast(FileTreeEntry::*)()>(&FileTreeEntry::parent)) + .def("path", &FileTreeEntry::path, bpy::arg("sep") = "\\") + .def("pathFrom", &FileTreeEntry::pathFrom, bpy::arg("sep") = "\\") // Mutable operation: .def("setTime", &FileTreeEntry::setTime) @@ -986,7 +986,7 @@ BOOST_PYTHON_MODULE(mobase) .def("exists", static_cast(&IFileTree::exists), (bpy::arg("type") = IFileTree::FILE_OR_DIRECTORY)) .def("find", static_cast(IFileTree::*)(QString, IFileTree::FileTypes)>(&IFileTree::find), bpy::arg("type") = IFileTree::FILE_OR_DIRECTORY, bpy::return_value_policy>()) - .def("getPathTo", &IFileTree::pathTo, bpy::arg("sep") = "\\") + .def("pathTo", &IFileTree::pathTo, bpy::arg("sep") = "\\") // Kind-of-static operations: .def("createOrphanTree", &IFileTree::createOrphanTree, bpy::arg("name") = "") From f1d5a1f31b2b7137394ef313ad62fef9e740be83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sat, 2 May 2020 13:22:55 +0200 Subject: [PATCH 012/341] Improve GuessedValue binding and switch to upper-case for enumerations in Python. --- src/runner/pythonrunner.cpp | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 7b42272f..e9ecc94b 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -1117,12 +1117,12 @@ BOOST_PYTHON_MODULE(mobase) ; bpy::enum_("GuessQuality") - .value("invalid", MOBase::GUESS_INVALID) - .value("fallback", MOBase::GUESS_FALLBACK) - .value("good", MOBase::GUESS_GOOD) - .value("meta", MOBase::GUESS_META) - .value("preset", MOBase::GUESS_PRESET) - .value("user", MOBase::GUESS_USER) + .value("INVALID", MOBase::GUESS_INVALID) + .value("FALLBACK", MOBase::GUESS_FALLBACK) + .value("GOOD", MOBase::GUESS_GOOD) + .value("META", MOBase::GUESS_META) + .value("PRESET", MOBase::GUESS_PRESET) + .value("USER", MOBase::GUESS_USER) ; bpy::class_, boost::noncopyable>("GuessedString", bpy::no_init) @@ -1132,6 +1132,14 @@ BOOST_PYTHON_MODULE(mobase) .def("update", static_cast& (GuessedValue::*)(const QString&, EGuessQuality)>(&GuessedValue::update), bpy::return_self<>()) + + // Methods to simulate the assignment operator: + .def("reset", +[](GuessedValue* gv) { *gv = GuessedValue(); }, bpy::return_self<>()) + .def("reset", +[](GuessedValue* gv, const QString& value, EGuessQuality eq) { *gv = GuessedValue(value, eq); }, bpy::return_self<>()) + + // Use an intermediate lambda to avoid having to register the std::function conversion: + .def("setFilter", +[](GuessedValue* gv, bpy::object fn) { gv->setFilter(fn); }) + .def("variants", &MOBase::GuessedValue::variants, bpy::return_value_policy()) .def("__str__", &MOBase::GuessedValue::operator const QString&, bpy::return_value_policy()) ; @@ -1285,11 +1293,11 @@ BOOST_PYTHON_MODULE(mobase) bpy::register_tuple, QString, int>>(); register_implicit_variant, boost::tuple, QString, int>>(); bpy::enum_("InstallResult") - .value("success", MOBase::IPluginInstaller::RESULT_SUCCESS) - .value("failed", MOBase::IPluginInstaller::RESULT_FAILED) - .value("canceled", MOBase::IPluginInstaller::RESULT_CANCELED) - .value("manualRequested", MOBase::IPluginInstaller::RESULT_MANUALREQUESTED) - .value("notAttempted", MOBase::IPluginInstaller::RESULT_NOTATTEMPTED) + .value("SUCCESS", MOBase::IPluginInstaller::RESULT_SUCCESS) + .value("FAILED", MOBase::IPluginInstaller::RESULT_FAILED) + .value("CANCELED", MOBase::IPluginInstaller::RESULT_CANCELED) + .value("MANUAL_REQUESTED", MOBase::IPluginInstaller::RESULT_MANUALREQUESTED) + .value("NOT_ATTEMPTED", MOBase::IPluginInstaller::RESULT_NOTATTEMPTED) ; bpy::class_("IPluginInstallerSimple") From 53b91f00925446e480f040f886065270a624e4d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sat, 2 May 2020 13:28:08 +0200 Subject: [PATCH 013/341] Update exposed method names for installer to be consistent with others. --- src/runner/pythonrunner.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index e9ecc94b..e8e39776 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -1302,16 +1302,16 @@ BOOST_PYTHON_MODULE(mobase) bpy::class_("IPluginInstallerSimple") .def("install", &IPluginInstallerSimple::install) - .def("getParentWidget", &IPluginInstallerSimpleWrapper::parentWidget, bpy::return_value_policy()) - .def("getManager", &IPluginInstallerSimpleWrapper::manager, bpy::return_value_policy()) + .def("parentWidget", &IPluginInstallerSimpleWrapper::parentWidget, bpy::return_value_policy()) + .def("manager", &IPluginInstallerSimpleWrapper::manager, bpy::return_value_policy()) ; bpy::class_("IPluginInstallerCustom") .def("isArchiveSupported", &IPluginInstallerCustom::isArchiveSupported) .def("supportedExtensions", &IPluginInstallerCustom::supportedExtensions) .def("install", &IPluginInstallerCustom::install) - .def("getParentWidget", &IPluginInstallerCustomWrapper::parentWidget, bpy::return_value_policy()) - .def("getManager", &IPluginInstallerCustomWrapper::manager, bpy::return_value_policy()) + .def("parentWidget", &IPluginInstallerCustomWrapper::parentWidget, bpy::return_value_policy()) + .def("manager", &IPluginInstallerCustomWrapper::manager, bpy::return_value_policy()) ; bpy::class_("IPluginModPage") From 9f5dab285d643b3c72e7785e6ec95b595e6ee66d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sat, 2 May 2020 15:04:44 +0200 Subject: [PATCH 014/341] Fix return_value_policy for methods returning QWidget. --- src/runner/pythonrunner.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index e8e39776..bb3a5e05 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -1302,7 +1302,7 @@ BOOST_PYTHON_MODULE(mobase) bpy::class_("IPluginInstallerSimple") .def("install", &IPluginInstallerSimple::install) - .def("parentWidget", &IPluginInstallerSimpleWrapper::parentWidget, bpy::return_value_policy()) + .def("parentWidget", &IPluginInstallerSimpleWrapper::parentWidget, bpy::return_value_policy()) .def("manager", &IPluginInstallerSimpleWrapper::manager, bpy::return_value_policy()) ; @@ -1310,7 +1310,7 @@ BOOST_PYTHON_MODULE(mobase) .def("isArchiveSupported", &IPluginInstallerCustom::isArchiveSupported) .def("supportedExtensions", &IPluginInstallerCustom::supportedExtensions) .def("install", &IPluginInstallerCustom::install) - .def("parentWidget", &IPluginInstallerCustomWrapper::parentWidget, bpy::return_value_policy()) + .def("parentWidget", &IPluginInstallerSimpleWrapper::parentWidget, bpy::return_value_policy()) .def("manager", &IPluginInstallerCustomWrapper::manager, bpy::return_value_policy()) ; From 817f8d72388567103d291e72cf7a75c63b254285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sat, 2 May 2020 15:05:24 +0200 Subject: [PATCH 015/341] Fix variants() method of GuessedValue. --- src/runner/pythonrunner.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index bb3a5e05..31fd0f12 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -1140,7 +1140,15 @@ BOOST_PYTHON_MODULE(mobase) // Use an intermediate lambda to avoid having to register the std::function conversion: .def("setFilter", +[](GuessedValue* gv, bpy::object fn) { gv->setFilter(fn); }) - .def("variants", &MOBase::GuessedValue::variants, bpy::return_value_policy()) + // Exposing the set does not work, but even if it worked, we would lose the order since it would be + // converted to a python set() so we expose a cusotm iterator. This works because variants() returns + // a reference, otherwize this would be more complex to do (also, needs to use references instead of + // pointers instead of the lambda due to the range() requirements): + .def("variants", bpy::range< + bpy::return_value_policy>( + +[](GuessedValue &gv) { return std::begin(gv.variants()); }, + +[](GuessedValue &gv) { return std::end(gv.variants()); } + )) .def("__str__", &MOBase::GuessedValue::operator const QString&, bpy::return_value_policy()) ; From 70172ddda6716b703cc44f6e7dabcc88649f5dca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sat, 2 May 2020 15:34:06 +0200 Subject: [PATCH 016/341] Add possibility to create GuessedString from python. --- src/runner/pythonrunner.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 31fd0f12..a2e0e098 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -1125,7 +1125,9 @@ BOOST_PYTHON_MODULE(mobase) .value("USER", MOBase::GUESS_USER) ; - bpy::class_, boost::noncopyable>("GuessedString", bpy::no_init) + bpy::class_, boost::noncopyable>("GuessedString") + .def(bpy::init<>()) + .def(bpy::init()) .def("update", static_cast& (GuessedValue::*)(const QString&)>(&GuessedValue::update), bpy::return_self<>()) @@ -1136,6 +1138,7 @@ BOOST_PYTHON_MODULE(mobase) // Methods to simulate the assignment operator: .def("reset", +[](GuessedValue* gv) { *gv = GuessedValue(); }, bpy::return_self<>()) .def("reset", +[](GuessedValue* gv, const QString& value, EGuessQuality eq) { *gv = GuessedValue(value, eq); }, bpy::return_self<>()) + .def("reset", +[](GuessedValue* gv, const GuessedValue& other) { *gv = other; }, bpy::return_self<>()) // Use an intermediate lambda to avoid having to register the std::function conversion: .def("setFilter", +[](GuessedValue* gv, bpy::object fn) { gv->setFilter(fn); }) From 8a286f9cfc4bab3ef890edf4a28897c312fb119f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sun, 3 May 2020 12:57:01 +0200 Subject: [PATCH 017/341] Change Functor_converter to use a function-type template argument. --- src/runner/pythonrunner.cpp | 48 ++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index a2e0e098..4696d82b 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -675,9 +675,12 @@ int getArgCount(PyObject *object) { return result; } +template +struct Functor_converter; + template -struct Functor_converter +struct Functor_converter { struct FunctorWrapper @@ -875,10 +878,10 @@ BOOST_PYTHON_MODULE(mobase) // TODO: ISaveGameInfoWidget bindings - Functor_converter(); - Functor_converter(); - Functor_converter(); - Functor_converter(); + Functor_converter(); + Functor_converter(); + Functor_converter(); + Functor_converter(); bpy::class_("IOrganizer") .def("createNexusBridge", bpy::pure_virtual(&IOrganizer::createNexusBridge), bpy::return_value_policy()) @@ -927,6 +930,9 @@ BOOST_PYTHON_MODULE(mobase) bpy::register_ptr_to_python>(); bpy::register_ptr_to_python>(); + // For removeIf: + Functor_converter const&)>(); + // FileTreeEntry Scope: auto fileTreeEntryClass = bpy::class_, boost::noncopyable>("FileTreeEntry", bpy::no_init); { @@ -1017,9 +1023,7 @@ BOOST_PYTHON_MODULE(mobase) .def("clear", &IFileTree::clear) .def("removeAll", &IFileTree::removeAll) - .def("removeIf", +[](IFileTree* p, boost::python::object fn) { - return p->removeIf(fn); - }) + .def("removeIf", &IFileTree::removeIf) // Special methods: .def("__getitem__", static_cast(IFileTree::*)(std::size_t)>(&IFileTree::at), @@ -1125,6 +1129,11 @@ BOOST_PYTHON_MODULE(mobase) .value("USER", MOBase::GUESS_USER) ; + // For setFilter, temporarily since the reference does not allow + // python plugin to update the name: + register_implicit_variant(); + Functor_converter(QString const&)>(); + bpy::class_, boost::noncopyable>("GuessedString") .def(bpy::init<>()) .def(bpy::init()) @@ -1141,7 +1150,24 @@ BOOST_PYTHON_MODULE(mobase) .def("reset", +[](GuessedValue* gv, const GuessedValue& other) { *gv = other; }, bpy::return_self<>()) // Use an intermediate lambda to avoid having to register the std::function conversion: - .def("setFilter", +[](GuessedValue* gv, bpy::object fn) { gv->setFilter(fn); }) + .def("setFilter", +[](GuessedValue* gv, std::function(QString const&)> fn) { + gv->setFilter([fn](QString& s) { + auto ret = fn(s); + return boost::apply_visitor([&s](auto v) { + qDebug() << "In visitor:" << typeid(v).name(); + if constexpr (std::is_same_v) { + s = v; + return true; + } + else if constexpr (std::is_same_v) { + return v; + } + else { + static_assert("Incorrect visitor."); + } + }, ret); + }); + }) // Exposing the set does not work, but even if it worked, we would lose the order since it would be // converted to a python set() so we expose a cusotm iterator. This works because variants() returns @@ -1157,7 +1183,7 @@ BOOST_PYTHON_MODULE(mobase) bpy::to_python_converter>(); QFlags_from_python_obj(); - Functor_converter(); // converter for the onRefreshed-callback + Functor_converter(); // converter for the onRefreshed-callback bpy::enum_("PluginState") .value("missing", IPluginList::STATE_MISSING) @@ -1181,7 +1207,7 @@ BOOST_PYTHON_MODULE(mobase) bpy::to_python_converter>(); QFlags_from_python_obj(); - Functor_converter(); // converter for the onModStateChanged-callback + Functor_converter(); // converter for the onModStateChanged-callback bpy::enum_("ModState") .value("exists", IModList::STATE_EXISTS) From f83ed392872f8771f3d49771ca31f3b43109567b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sun, 3 May 2020 12:57:23 +0200 Subject: [PATCH 018/341] Modify FileTypes enumeration for FileTreeEntry. --- src/runner/pythonrunner.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 4696d82b..68a68519 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -938,9 +938,8 @@ BOOST_PYTHON_MODULE(mobase) { bpy::scope scope = fileTreeEntryClass; - - bpy::enum_("FileType") + bpy::enum_("FileTypes") .value("FILE_OR_DIRECTORY", FileTreeEntry::FILE_OR_DIRECTORY) .value("FILE", FileTreeEntry::FILE) .value("DIRECTORY", FileTreeEntry::DIRECTORY) @@ -951,7 +950,8 @@ BOOST_PYTHON_MODULE(mobase) .def("isFile", &FileTreeEntry::isFile) .def("isDir", &FileTreeEntry::isDir) - .def("fileType", &FileTreeEntry::fileType) + // Forcing the conversion to FileTypeS to avoid having to expose FileType in python: + .def("fileType", +[](FileTreeEntry* p) { return FileTreeEntry::FileTypes{ p->fileType() }; }) // This should probably not be exposed in python since we provide automatic downcast: // .def("getTree", static_cast(FileTreeEntry::*)()>(&FileTreeEntry::astree)) .def("name", &FileTreeEntry::name) From 8b118ad89870202776eca8111bff1d6e00115421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sun, 3 May 2020 13:38:47 +0200 Subject: [PATCH 019/341] Better handling of boost::variant. --- src/runner/pythonrunner.cpp | 30 +++-------- src/runner/variant_helper.h | 99 +++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 22 deletions(-) create mode 100644 src/runner/variant_helper.h diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 68a68519..562ccf3d 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -26,7 +26,9 @@ #ifndef Q_MOC_RUN #include #include + #include "tuple_helper.h" +#include "variant_helper.h" #endif MOBase::IOrganizer *s_Organizer = nullptr; @@ -772,20 +774,6 @@ struct DowncastReturn { }; -/** - * @brief Register implicit conversion from any of the decayed type from the variant - * to the variant type. - */ -template -void register_implicit_variant() { - mp11::mp_for_each< - mp11::mp_list> ( - [](auto t) { - // take advantage of built-in conversions - bpy::implicitly_convertible, boost::variant>(); - }); -} - BOOST_PYTHON_MODULE(mobase) { @@ -1129,10 +1117,9 @@ BOOST_PYTHON_MODULE(mobase) .value("USER", MOBase::GUESS_USER) ; - // For setFilter, temporarily since the reference does not allow - // python plugin to update the name: - register_implicit_variant(); - Functor_converter(QString const&)>(); + // For setFilter: + bpy::register_variant>(); + Functor_converter(QString const&)>(); bpy::class_, boost::noncopyable>("GuessedString") .def(bpy::init<>()) @@ -1150,11 +1137,10 @@ BOOST_PYTHON_MODULE(mobase) .def("reset", +[](GuessedValue* gv, const GuessedValue& other) { *gv = other; }, bpy::return_self<>()) // Use an intermediate lambda to avoid having to register the std::function conversion: - .def("setFilter", +[](GuessedValue* gv, std::function(QString const&)> fn) { + .def("setFilter", +[](GuessedValue* gv, std::function(QString const&)> fn) { gv->setFilter([fn](QString& s) { auto ret = fn(s); return boost::apply_visitor([&s](auto v) { - qDebug() << "In visitor:" << typeid(v).name(); if constexpr (std::is_same_v) { s = v; return true; @@ -1170,7 +1156,7 @@ BOOST_PYTHON_MODULE(mobase) }) // Exposing the set does not work, but even if it worked, we would lose the order since it would be - // converted to a python set() so we expose a cusotm iterator. This works because variants() returns + // converted to a python set() so we expose a custom iterator. This works because variants() returns // a reference, otherwize this would be more complex to do (also, needs to use references instead of // pointers instead of the lambda due to the range() requirements): .def("variants", bpy::range< @@ -1328,7 +1314,7 @@ BOOST_PYTHON_MODULE(mobase) ; bpy::register_tuple, QString, int>>(); - register_implicit_variant, boost::tuple, QString, int>>(); + bpy::register_variant, boost::tuple, QString, int>>>(); bpy::enum_("InstallResult") .value("SUCCESS", MOBase::IPluginInstaller::RESULT_SUCCESS) .value("FAILED", MOBase::IPluginInstaller::RESULT_FAILED) diff --git a/src/runner/variant_helper.h b/src/runner/variant_helper.h new file mode 100644 index 00000000..50d238ea --- /dev/null +++ b/src/runner/variant_helper.h @@ -0,0 +1,99 @@ +#ifndef VARIANT_HELPER_H +#define VARIANT_HELPER_H + +#include +#include +#include //len function +#include +#include + +/** + * Creates boost::variant from python object. Greatly inspired by register_tuple<>. + */ + +namespace boost { + namespace python { + + template + struct variant_from_python; + + template + struct variant_from_python> { + + using variant_type = boost::variant; + + static void* convertible(PyObject* py_obj) { + + python::object obj(handle<>(borrowed(py_obj))); + + if (impl::convertible(obj)) { + return py_obj; + } + else { + return 0; + } + } + + static void construct(PyObject* py_obj, converter::rvalue_from_python_stage1_data* data) { + typedef converter::rvalue_from_python_storage storage_t; + storage_t* the_storage = reinterpret_cast(data); + void* memory_chunk = the_storage->storage.bytes; + variant_type* c_variant = new (memory_chunk) variant_type(); + data->convertible = memory_chunk; + + python::object obj(handle<>(borrowed(py_obj))); + impl::construct(obj, *c_variant); + } + + private: + + template + struct impl; + + template <> + struct impl<> { + static bool convertible(const python::object& obj) { return false; } + static void construct(const python::object& obj, variant_type& c_variant) {} + }; + + template + struct impl { + + static bool convertible(const python::object& obj) { + + extract type_checker(obj); + if (type_checker.check()) { + return true; + } + else { + return impl::convertible(obj); + } + } + + static void construct(const python::object& obj, variant_type& c_variant) { + + extract type_checker(obj); + + if (type_checker.check()) { + c_variant = type_checker(); + } + else { + impl::construct(obj, c_variant); + } + } + + }; + + }; + + template< class TVariant> + void register_variant() { + converter::registry::push_back(&variant_from_python::convertible + , &variant_from_python::construct + , type_id()); + }; + + } +} //boost::python + +#endif//TUPLES_HPP_16_JAN_2007 \ No newline at end of file From 221949f48b4a6e7df95bab19bc95aab7fc5c4192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sun, 3 May 2020 13:39:04 +0200 Subject: [PATCH 020/341] Fix issue with Functor_converter when return type was not bool or void. --- src/runner/pythonrunner.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 562ccf3d..a00ca9ec 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -697,7 +697,12 @@ struct Functor_converter RET operator()(const PARAMS &...params) { GILock lock; - return (RET) m_Callable(params...); + if constexpr (std::is_same_v) { + m_Callable(params...); + } + else { + return bpy::extract(m_Callable(params...)); + } } boost::python::object m_Callable; From 585296d527ed06921eca5b259f9ba70f8a0abcf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sun, 3 May 2020 18:24:32 +0200 Subject: [PATCH 021/341] Fix issue with shared pointer being registered twice. --- src/runner/pythonrunner.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index a00ca9ec..4b16fb9d 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -772,11 +772,17 @@ template struct DowncastReturn { template - struct apply { - static_assert(std::is_convertible_v>); + struct apply_; + + template + struct apply_> { + static_assert(std::is_convertible_v, std::shared_ptr>); using type = DowncastConverter; }; + template + using apply = apply_>; + }; @@ -927,7 +933,7 @@ BOOST_PYTHON_MODULE(mobase) Functor_converter const&)>(); // FileTreeEntry Scope: - auto fileTreeEntryClass = bpy::class_, boost::noncopyable>("FileTreeEntry", bpy::no_init); + auto fileTreeEntryClass = bpy::class_("FileTreeEntry", bpy::no_init); { bpy::scope scope = fileTreeEntryClass; From 5cd7917709c41184b4daa7b1059a46f35df3c833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sun, 3 May 2020 18:25:16 +0200 Subject: [PATCH 022/341] Add to_python to register_variant.h. --- src/runner/variant_helper.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/runner/variant_helper.h b/src/runner/variant_helper.h index 50d238ea..83fbcaaa 100644 --- a/src/runner/variant_helper.h +++ b/src/runner/variant_helper.h @@ -14,6 +14,24 @@ namespace boost { namespace python { + template + struct to_py_variant; + + template + struct to_py_variant> { + + using variant_type = boost::variant; + + static PyObject* convert(const variant_type& c_variant) { + object value = boost::apply_visitor([](auto const& value) { + return object{ value }; + }, c_variant); + //create Python object from the list + return incref(value.ptr()); + } + + }; + template struct variant_from_python; @@ -88,6 +106,9 @@ namespace boost { template< class TVariant> void register_variant() { + + to_python_converter>(); + converter::registry::push_back(&variant_from_python::convertible , &variant_from_python::construct , type_id()); From 473cef62173c3f44ff40007a80741bd0c41ccb95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sun, 3 May 2020 18:25:35 +0200 Subject: [PATCH 023/341] Add utility package with register_map for std::map. --- src/runner/pythonutils.h | 58 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/runner/pythonutils.h diff --git a/src/runner/pythonutils.h b/src/runner/pythonutils.h new file mode 100644 index 00000000..dab156da --- /dev/null +++ b/src/runner/pythonutils.h @@ -0,0 +1,58 @@ +#ifndef PYTHONRUNNER_UTILS_H +#define PYTHONRUNNER_UTILS_H + +#include + +namespace utils { + + template + struct map_to_python { + static PyObject* convert(const Map& map) { + boost::python::dict result; + for (auto& entry : map) { + result[boost::python::object{ entry.first }] = boost::python::object{ entry.second }; + } + return boost::python::incref(result.ptr()); + } + }; + + template + struct map_from_python { + + using key_type = typename Map::key_type; + using value_type = typename Map::mapped_type; + + static void* convertible(PyObject* objPtr) { + return PyDict_Check(objPtr) ? objPtr : nullptr; + } + + static void construct(PyObject* objPtr, boost::python::converter::rvalue_from_python_stage1_data* data) { + void* storage = ((boost::python::converter::rvalue_from_python_storage*)data)->storage.bytes; + Map* result = new (storage) Map(); + boost::python::dict source(boost::python::handle<>(boost::python::borrowed(objPtr))); + boost::python::list keys = source.keys(); + int len = boost::python::len(keys); + for (int i = 0; i < len; ++i) { + boost::python::object pyKey = keys[i]; + (*result)[boost::python::extract(pyKey)] = boost::python::extract(source[pyKey]); + } + + data->convertible = storage; + } + }; + + template + void register_map() { + + boost::python::to_python_converter>(); + + boost::python::converter::registry::push_back( + &map_from_python::convertible + , &map_from_python::construct + , boost::python::type_id()); + }; + + +} + +#endif \ No newline at end of file From af37a451f92cad12a901e317ace49f77fde8b2ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sun, 3 May 2020 18:26:11 +0200 Subject: [PATCH 024/341] Better variant handling for IFileTree::merge. --- src/runner/pythonrunner.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 4b16fb9d..ec48f5e3 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -29,6 +29,7 @@ #include "tuple_helper.h" #include "variant_helper.h" +#include "pythonutils.h" #endif MOBase::IOrganizer *s_Organizer = nullptr; @@ -971,6 +972,9 @@ BOOST_PYTHON_MODULE(mobase) ; } + utils::register_map(); + bpy::register_variant>(); + // IFileTree scope: auto iFileTreeClass = bpy::class_, boost::noncopyable>("IFileTree", bpy::no_init);; { @@ -1003,16 +1007,16 @@ BOOST_PYTHON_MODULE(mobase) IFileTree* p, std::shared_ptr entry, IFileTree::InsertPolicy insertPolicy) { return p->insert(entry, insertPolicy) != p->end(); }, bpy::arg("policy") = IFileTree::InsertPolicy::FAIL_IF_EXISTS) - .def("merge", +[](IFileTree* p, std::shared_ptr other, bool returnOverwrites) -> bpy::object { + .def("merge", +[](IFileTree* p, std::shared_ptr other, bool returnOverwrites) -> boost::variant { IFileTree::OverwritesType overwrites; auto result = p->merge(other, returnOverwrites ? &overwrites : nullptr); if (result == IFileTree::MERGE_FAILED) { - return bpy::object{ false }; + return { false }; } if (returnOverwrites) { - return bpy::object{ overwrites }; + return { overwrites }; } - return bpy::object{ result }; + return { result }; }, bpy::arg("overwrites") = false) .def("move", &IFileTree::move, bpy::arg("policy") = IFileTree::InsertPolicy::FAIL_IF_EXISTS) From 1ea006b74ec1c199920d44b08b672b0b97c64690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sun, 3 May 2020 18:26:27 +0200 Subject: [PATCH 025/341] Add bindings for FileInfo. --- src/runner/pythonrunner.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index ec48f5e3..a0c58a9b 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -883,6 +883,12 @@ BOOST_PYTHON_MODULE(mobase) Functor_converter(); Functor_converter(); + bpy::class_("FileInfo", bpy::init<>()) + .def_readwrite("filePath", &IOrganizer::FileInfo::filePath) + .def_readwrite("archive", &IOrganizer::FileInfo::archive) + .def_readwrite("origins", &IOrganizer::FileInfo::origins) + ; + bpy::class_("IOrganizer") .def("createNexusBridge", bpy::pure_virtual(&IOrganizer::createNexusBridge), bpy::return_value_policy()) .def("profileName", bpy::pure_virtual(&IOrganizer::profileName)) From 61ad2ef2b6732670f28b3190073e42259d790d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sun, 3 May 2020 18:27:26 +0200 Subject: [PATCH 026/341] Clean bindings for IFileTree and FileTreeEntry. --- src/runner/pythonrunner.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index a0c58a9b..f449a7b4 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -972,9 +972,16 @@ BOOST_PYTHON_MODULE(mobase) .def("detach", &FileTreeEntry::detach) .def("moveTo", &FileTreeEntry::moveTo) + // Special methods: + .def("__eq__", +[](const FileTreeEntry* entry, QString other) { + return entry->compare(other) == 0; + }) + .def("__eq__", +[](const FileTreeEntry* entry, std::shared_ptr other) { + return entry == other.get(); + }) + // Special methods for debug: - .def("__str__", &FileTreeEntry::name) - .def("__repr__", +[](const FileTreeEntry* entry) { return "FileTreeEntry(" + entry->name() + ")"; }) + .def("__repr__", +[](const FileTreeEntry* entry) { return "FileTreeEntry(\"" + entry->name() + "\")"; }) ; } @@ -1042,8 +1049,7 @@ BOOST_PYTHON_MODULE(mobase) static_cast(&IFileTree::end))) .def("__len__", &IFileTree::size) .def("__bool__", +[](const IFileTree* tree) { return !tree->empty(); }) - .def("__str__", &FileTreeEntry::name) - .def("__repr__", +[](const IFileTree* entry) { return "IFileTree(" + entry->name() + ")"; }) + .def("__repr__", +[](const IFileTree* entry) { return "IFileTree(\"" + entry->name() + "\")"; }) ; } From aac45a39e2b0a05118b9041dc92406036fb4358c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sun, 3 May 2020 20:16:57 +0200 Subject: [PATCH 027/341] Make register_variant work for all variant types. --- src/runner/variant_helper.h | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/runner/variant_helper.h b/src/runner/variant_helper.h index 83fbcaaa..618160f6 100644 --- a/src/runner/variant_helper.h +++ b/src/runner/variant_helper.h @@ -14,15 +14,10 @@ namespace boost { namespace python { - template - struct to_py_variant; - - template - struct to_py_variant> { - - using variant_type = boost::variant; + template + struct to_py_variant { - static PyObject* convert(const variant_type& c_variant) { + static PyObject* convert(const TVariant& c_variant) { object value = boost::apply_visitor([](auto const& value) { return object{ value }; }, c_variant); @@ -35,10 +30,10 @@ namespace boost { template struct variant_from_python; - template - struct variant_from_python> { + template