From d9f38fca5ffe29136cd6f390e318ad8455864582 Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Fri, 18 Oct 2024 12:23:34 +1100 Subject: [PATCH] PyAPI: support Python 3.13 - `_PySet_NextEntry` has been removed, use generic iterator access which will has some additional overhead as it needs to create an iterator to access the values. - Add v3.13 compatibility defines to account for renaming: _PyObject_LookupAttr -> PyObject_GetOptionalAttr _PyLong_AsInt -> PyLong_AsInt - Unfortunately use of Python's internal API needs to be used to inspect op-codes in `bpy_driver.cc`. Testing GLTF/FBX IO there isn't any significant performance impact from these changes. Resolves #123871. --- .../blender/python/bmesh/bmesh_py_ops_call.cc | 16 +- source/blender/python/generic/py_capi_rna.cc | 140 ++++++++++-------- .../blender/python/generic/py_capi_utils.cc | 56 ++++--- .../blender/python/generic/py_capi_utils.hh | 4 + .../blender/python/generic/python_compat.hh | 10 +- .../blender/python/gpu/gpu_py_framebuffer.cc | 6 +- source/blender/python/intern/bpy.cc | 48 +++--- source/blender/python/intern/bpy_driver.cc | 37 ++++- .../python/intern/bpy_library_write.cc | 27 ++-- source/blender/python/intern/bpy_rna.cc | 6 +- 10 files changed, 218 insertions(+), 132 deletions(-) diff --git a/source/blender/python/bmesh/bmesh_py_ops_call.cc b/source/blender/python/bmesh/bmesh_py_ops_call.cc index d5a2e6b02ad..624e4d02e98 100644 --- a/source/blender/python/bmesh/bmesh_py_ops_call.cc +++ b/source/blender/python/bmesh/bmesh_py_ops_call.cc @@ -548,11 +548,12 @@ static int bpy_slot_from_py(BMesh *bm, break; } case BMO_OP_SLOT_SUBTYPE_MAP_EMPTY: { - if (PySet_Size(value) > 0) { + if (PySet_GET_SIZE(value) > 0) { + PyObject *it = PyObject_GetIter(value); PyObject *arg_key; - Py_ssize_t arg_pos = 0; - Py_ssize_t arg_hash = 0; - while (_PySet_NextEntry(value, &arg_pos, &arg_key, &arg_hash)) { + while ((arg_key = PyIter_Next(it))) { + /* Borrow from the set. */ + Py_DECREF(arg_key); if (bpy_slot_from_py_elem_check((BPy_BMElem *)arg_key, bm, @@ -561,11 +562,16 @@ static int bpy_slot_from_py(BMesh *bm, slot_name, "invalid key in set") == -1) { - return -1; /* error is set in bpy_slot_from_py_elem_check() */ + /* Error is set in #bpy_slot_from_py_elem_check(). */ + break; } BMO_slot_map_empty_insert(bmop, slot, ((BPy_BMElem *)arg_key)->ele); } + Py_DECREF(it); + if (arg_key) { + return -1; + } } break; } diff --git a/source/blender/python/generic/py_capi_rna.cc b/source/blender/python/generic/py_capi_rna.cc index 11827fa0836..a41e474f619 100644 --- a/source/blender/python/generic/py_capi_rna.cc +++ b/source/blender/python/generic/py_capi_rna.cc @@ -75,60 +75,66 @@ BLI_bitmap *pyrna_enum_bitmap_from_set(const EnumPropertyItem *items, int bitmap_size, const char *error_prefix) { - /* Set looping. */ - Py_ssize_t pos = 0; - Py_ssize_t hash = 0; - PyObject *key; - + BLI_assert(PySet_Check(value)); BLI_bitmap *bitmap = BLI_BITMAP_NEW(bitmap_size, __func__); - while (_PySet_NextEntry(value, &pos, &key, &hash)) { - const char *param = PyUnicode_AsUTF8(key); - if (param == nullptr) { - PyErr_Format(PyExc_TypeError, - "%.200s expected a string, not %.200s", - error_prefix, - Py_TYPE(key)->tp_name); - goto error; - } + if (PySet_GET_SIZE(value) > 0) { + /* Set looping. */ + PyObject *it = PyObject_GetIter(value); + PyObject *key; + while ((key = PyIter_Next(it))) { + /* Borrow from the set. */ + Py_DECREF(key); - int ret; - if (pyrna_enum_value_from_id(items, param, &ret, error_prefix) == -1) { - goto error; - } + const char *param = PyUnicode_AsUTF8(key); + if (param == nullptr) { + PyErr_Format(PyExc_TypeError, + "%.200s expected a string, not %.200s", + error_prefix, + Py_TYPE(key)->tp_name); + break; + } - int index = ret; + int ret; + if (pyrna_enum_value_from_id(items, param, &ret, error_prefix) == -1) { + break; + } - if (type_convert_sign) { - if (type_size == 2) { - union { - signed short as_signed; - ushort as_unsigned; - } ret_convert; - ret_convert.as_signed = (signed short)ret; - index = int(ret_convert.as_unsigned); - } - else if (type_size == 1) { - union { - signed char as_signed; - uchar as_unsigned; - } ret_convert; - ret_convert.as_signed = (signed char)ret; - index = int(ret_convert.as_unsigned); - } - else { - BLI_assert_unreachable(); + int index = ret; + + if (type_convert_sign) { + if (type_size == 2) { + union { + signed short as_signed; + ushort as_unsigned; + } ret_convert; + ret_convert.as_signed = (signed short)ret; + index = int(ret_convert.as_unsigned); + } + else if (type_size == 1) { + union { + signed char as_signed; + uchar as_unsigned; + } ret_convert; + ret_convert.as_signed = (signed char)ret; + index = int(ret_convert.as_unsigned); + } + else { + BLI_assert_unreachable(); + } } + BLI_assert(index < bitmap_size); + BLI_BITMAP_ENABLE(bitmap, index); + } + Py_DECREF(it); + + if (key) { + MEM_freeN(bitmap); + bitmap = nullptr; } - BLI_assert(index < bitmap_size); - BLI_BITMAP_ENABLE(bitmap, index); } return bitmap; - -error: - MEM_freeN(bitmap); - return nullptr; } int pyrna_enum_bitfield_from_set(const EnumPropertyItem *items, @@ -136,32 +142,40 @@ int pyrna_enum_bitfield_from_set(const EnumPropertyItem *items, int *r_value, const char *error_prefix) { + BLI_assert(PySet_Check(value)); /* Set of enum items, concatenate all values with OR. */ - int ret, flag = 0; - - /* Set looping. */ - Py_ssize_t pos = 0; - Py_ssize_t hash = 0; - PyObject *key; + int flag = 0; *r_value = 0; - while (_PySet_NextEntry(value, &pos, &key, &hash)) { - const char *param = PyUnicode_AsUTF8(key); + PyObject *key = nullptr; + if (PySet_GET_SIZE(value) > 0) { + /* Set looping. */ + PyObject *it = PyObject_GetIter(value); + while ((key = PyIter_Next(it))) { + /* Borrow from the set. */ + Py_DECREF(key); - if (param == nullptr) { - PyErr_Format(PyExc_TypeError, - "%.200s expected a string, not %.200s", - error_prefix, - Py_TYPE(key)->tp_name); + const char *param = PyUnicode_AsUTF8(key); + if (param == nullptr) { + PyErr_Format(PyExc_TypeError, + "%.200s expected a string, not %.200s", + error_prefix, + Py_TYPE(key)->tp_name); + break; + } + + int ret; + if (pyrna_enum_value_from_id(items, param, &ret, error_prefix) == -1) { + break; + } + + flag |= ret; + } + Py_DECREF(it); + if (key) { return -1; } - - if (pyrna_enum_value_from_id(items, param, &ret, error_prefix) == -1) { - return -1; - } - - flag |= ret; } *r_value = flag; diff --git a/source/blender/python/generic/py_capi_utils.cc b/source/blender/python/generic/py_capi_utils.cc index 3311064e7c5..74c829d717e 100644 --- a/source/blender/python/generic/py_capi_utils.cc +++ b/source/blender/python/generic/py_capi_utils.cc @@ -39,6 +39,11 @@ # include "BLI_math_base.h" /* isfinite() */ #endif +#if PY_VERSION_HEX <= 0x030c0000 /* <=3.12 */ +# define PyLong_AsInt _PyLong_AsInt +# define PyUnicode_CompareWithASCIIString _PyUnicode_EqualToASCIIString +#endif + /* -------------------------------------------------------------------- */ /** \name Fast Python to C Array Conversion for Primitive Types * \{ */ @@ -874,10 +879,12 @@ static void pyc_exception_buffer_handle_system_exit() if (!PyErr_ExceptionMatches(PyExc_SystemExit)) { return; } - /* Inspecting, follow Python's logic in #_Py_HandleSystemExit & treat as a regular exception. */ +/* Inspecting, follow Python's logic in #_Py_HandleSystemExit & treat as a regular exception. */ +# if 0 /* FIXME: */ if (_Py_GetConfig()->inspect) { return; } +# endif /* NOTE(@ideasman42): A `SystemExit` exception will exit immediately (unless inspecting). * So print the error and exit now. Without this #PyErr_Display shows the error stack-trace @@ -1424,11 +1431,6 @@ int PyC_FlagSet_ToBitfield(const PyC_FlagSet *items, /* set of enum items, concatenate all values with OR */ int ret, flag = 0; - /* set looping */ - Py_ssize_t pos = 0; - Py_ssize_t hash = 0; - PyObject *key; - if (!PySet_Check(value)) { PyErr_Format(PyExc_TypeError, "%.200s expected a set, not %.200s", @@ -1439,22 +1441,32 @@ int PyC_FlagSet_ToBitfield(const PyC_FlagSet *items, *r_value = 0; - while (_PySet_NextEntry(value, &pos, &key, &hash)) { - const char *param = PyUnicode_AsUTF8(key); + if (PySet_GET_SIZE(value) > 0) { + PyObject *it = PyObject_GetIter(value); + PyObject *key; + while ((key = PyIter_Next(it))) { + /* Borrow from the set. */ + Py_DECREF(key); - if (param == nullptr) { - PyErr_Format(PyExc_TypeError, - "%.200s set must contain strings, not %.200s", - error_prefix, - Py_TYPE(key)->tp_name); + const char *param = PyUnicode_AsUTF8(key); + if (param == nullptr) { + PyErr_Format(PyExc_TypeError, + "%.200s set must contain strings, not %.200s", + error_prefix, + Py_TYPE(key)->tp_name); + break; + } + + if (PyC_FlagSet_ValueFromID(items, param, &ret, error_prefix) < 0) { + break; + } + + flag |= ret; + } + Py_DECREF(it); + if (key != nullptr) { return -1; } - - if (PyC_FlagSet_ValueFromID(items, param, &ret, error_prefix) < 0) { - return -1; - } - - flag |= ret; } *r_value = flag; @@ -1724,7 +1736,7 @@ static ulong pyc_Long_AsUnsignedLong(PyObject *value) int PyC_Long_AsBool(PyObject *value) { - const int test = _PyLong_AsInt(value); + const int test = PyLong_AsInt(value); if (UNLIKELY(test == -1 && PyErr_Occurred())) { return -1; } @@ -1737,7 +1749,7 @@ int PyC_Long_AsBool(PyObject *value) int8_t PyC_Long_AsI8(PyObject *value) { - const int test = _PyLong_AsInt(value); + const int test = PyLong_AsInt(value); if (UNLIKELY(test == -1 && PyErr_Occurred())) { return -1; } @@ -1750,7 +1762,7 @@ int8_t PyC_Long_AsI8(PyObject *value) int16_t PyC_Long_AsI16(PyObject *value) { - const int test = _PyLong_AsInt(value); + const int test = PyLong_AsInt(value); if (UNLIKELY(test == -1 && PyErr_Occurred())) { return -1; } diff --git a/source/blender/python/generic/py_capi_utils.hh b/source/blender/python/generic/py_capi_utils.hh index 4a641f78471..81740e0d0ad 100644 --- a/source/blender/python/generic/py_capi_utils.hh +++ b/source/blender/python/generic/py_capi_utils.hh @@ -339,7 +339,11 @@ uint64_t PyC_Long_AsU64(PyObject *value); /* inline so type signatures match as expected */ Py_LOCAL_INLINE(int32_t) PyC_Long_AsI32(PyObject *value) { +#if PY_VERSION_HEX <= 0x030c0000 /* <=3.12 */ return (int32_t)_PyLong_AsInt(value); +#else + return (int32_t)PyLong_AsInt(value); +#endif } Py_LOCAL_INLINE(int64_t) PyC_Long_AsI64(PyObject *value) { diff --git a/source/blender/python/generic/python_compat.hh b/source/blender/python/generic/python_compat.hh index f18d0fb59b3..e2799aeeb46 100644 --- a/source/blender/python/generic/python_compat.hh +++ b/source/blender/python/generic/python_compat.hh @@ -9,9 +9,17 @@ #pragma once +/* Removes `intialized` member from Python 3.13+. */ +#if PY_VERSION_HEX >= 0x030d0000 +# define PY_ARG_PARSER_HEAD_COMPAT() +#elif PY_VERSION_HEX >= 0x030c0000 /* Add `intialized` member for Python 3.12+. */ -#if PY_VERSION_HEX >= 0x030c0000 # define PY_ARG_PARSER_HEAD_COMPAT() 0, #else # define PY_ARG_PARSER_HEAD_COMPAT() #endif + +/* Python 3.13 made some changes, use the "new" names. */ +#if PY_VERSION_HEX < 0x030d0000 +# define PyObject_GetOptionalAttr _PyObject_LookupAttr +#endif diff --git a/source/blender/python/gpu/gpu_py_framebuffer.cc b/source/blender/python/gpu/gpu_py_framebuffer.cc index 899ad21f770..9bef53654df 100644 --- a/source/blender/python/gpu/gpu_py_framebuffer.cc +++ b/source/blender/python/gpu/gpu_py_framebuffer.cc @@ -286,14 +286,14 @@ static bool pygpu_framebuffer_new_parse_arg(PyObject *o, GPUAttachment *r_attach return false; } - if (c_texture && _PyUnicode_EqualToASCIIString(key, c_texture)) { + if (c_texture && PyUnicode_CompareWithASCIIString(key, c_texture)) { /* Compare only once. */ c_texture = nullptr; if (!bpygpu_ParseTexture(value, &tmp_attach.tex)) { return false; } } - else if (c_layer && _PyUnicode_EqualToASCIIString(key, c_layer)) { + else if (c_layer && PyUnicode_CompareWithASCIIString(key, c_layer)) { /* Compare only once. */ c_layer = nullptr; tmp_attach.layer = PyLong_AsLong(value); @@ -301,7 +301,7 @@ static bool pygpu_framebuffer_new_parse_arg(PyObject *o, GPUAttachment *r_attach return false; } } - else if (c_mip && _PyUnicode_EqualToASCIIString(key, c_mip)) { + else if (c_mip && PyUnicode_CompareWithASCIIString(key, c_mip)) { /* Compare only once. */ c_mip = nullptr; tmp_attach.mip = PyLong_AsLong(value); diff --git a/source/blender/python/intern/bpy.cc b/source/blender/python/intern/bpy.cc index 7cdd00fc7a3..42ea76e40ac 100644 --- a/source/blender/python/intern/bpy.cc +++ b/source/blender/python/intern/bpy.cc @@ -610,38 +610,40 @@ PyDoc_STRVAR( " :rtype: dict\n"); static PyObject *bpy_wm_capabilities(PyObject *self) { - static _Py_Identifier PyId_capabilities = {"_wm_capabilities_", -1}; - + PyObject *py_id_capabilities = PyUnicode_FromString("_wm_capabilities_"); PyObject *result = nullptr; - switch (_PyObject_LookupAttrId(self, &PyId_capabilities, &result)) { - case 1: - return result; - case 0: - break; - default: - /* Unlikely, but there may be an error, forward it. */ - return nullptr; - } + switch (PyObject_GetOptionalAttr(self, py_id_capabilities, &result)) { + case 1: { + result = PyDict_New(); - result = PyDict_New(); - - const eWM_CapabilitiesFlag flag = WM_capabilities_flag(); + const eWM_CapabilitiesFlag flag = WM_capabilities_flag(); #define SetFlagItem(x) \ PyDict_SetItemString(result, STRINGIFY(x), PyBool_FromLong((WM_CAPABILITY_##x) & flag)); - SetFlagItem(CURSOR_WARP); - SetFlagItem(WINDOW_POSITION); - SetFlagItem(PRIMARY_CLIPBOARD); - SetFlagItem(GPU_FRONT_BUFFER_READ); - SetFlagItem(CLIPBOARD_IMAGES); - SetFlagItem(DESKTOP_SAMPLE); - SetFlagItem(INPUT_IME); - SetFlagItem(TRACKPAD_PHYSICAL_DIRECTION); + SetFlagItem(CURSOR_WARP); + SetFlagItem(WINDOW_POSITION); + SetFlagItem(PRIMARY_CLIPBOARD); + SetFlagItem(GPU_FRONT_BUFFER_READ); + SetFlagItem(CLIPBOARD_IMAGES); + SetFlagItem(DESKTOP_SAMPLE); + SetFlagItem(INPUT_IME); + SetFlagItem(TRACKPAD_PHYSICAL_DIRECTION); #undef SetFlagItem + PyObject_SetAttr(self, py_id_capabilities, result); + break; + } + case 0: + BLI_assert(result != nullptr); + break; + default: + /* Unlikely, but there may be an error, forward it. */ + BLI_assert(result == nullptr); + break; + } - _PyObject_SetAttrId(self, &PyId_capabilities, result); + Py_DECREF(py_id_capabilities); return result; } diff --git a/source/blender/python/intern/bpy_driver.cc b/source/blender/python/intern/bpy_driver.cc index b32bfbc2a56..e589f0992e1 100644 --- a/source/blender/python/intern/bpy_driver.cc +++ b/source/blender/python/intern/bpy_driver.cc @@ -43,6 +43,13 @@ # include #endif +#if PY_VERSION_HEX >= 0x030d0000 /* >=3.13 */ +/* WARNING(@ideasman42): Using `Py_BUILD_CORE` is a last resort, + * the alternative would be not to inspect OP-CODES at all. */ +# define Py_BUILD_CORE +# include +#endif + PyObject *bpy_pydriver_Dict = nullptr; #ifdef USE_BYTECODE_WHITELIST @@ -375,7 +382,35 @@ static bool is_opcode_secure(const int opcode) OK_OP(LOAD_CONST) /* Ok because constants are accepted. */ OK_OP(LOAD_NAME) /* Ok, because `PyCodeObject.names` is checked. */ OK_OP(CALL) /* Ok, because we check its "name" before calling. */ - OK_OP(KW_NAMES) /* Ok, because it's used for calling functions with keyword arguments. */ +# if PY_VERSION_HEX >= 0x030d0000 + OK_OP(CALL_KW) /* Ok, because it's used for calling functions with keyword arguments. */ + + OK_OP(CALL_FUNCTION_EX); + + /* OK because the names are checked. */ + OK_OP(CALL_ALLOC_AND_ENTER_INIT) + OK_OP(CALL_BOUND_METHOD_EXACT_ARGS) + OK_OP(CALL_BOUND_METHOD_GENERAL) + OK_OP(CALL_BUILTIN_CLASS) + OK_OP(CALL_BUILTIN_FAST) + OK_OP(CALL_BUILTIN_FAST_WITH_KEYWORDS) + OK_OP(CALL_BUILTIN_O) + OK_OP(CALL_ISINSTANCE) + OK_OP(CALL_LEN) + OK_OP(CALL_LIST_APPEND) + OK_OP(CALL_METHOD_DESCRIPTOR_FAST) + OK_OP(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) + OK_OP(CALL_METHOD_DESCRIPTOR_NOARGS) + OK_OP(CALL_METHOD_DESCRIPTOR_O) + OK_OP(CALL_NON_PY_GENERAL) + OK_OP(CALL_PY_EXACT_ARGS) + OK_OP(CALL_PY_GENERAL) + OK_OP(CALL_STR_1) + OK_OP(CALL_TUPLE_1) + OK_OP(CALL_TYPE_1) +# else + OK_OP(KW_NAMES) /* Ok, because it's used for calling functions with keyword arguments. */ +# endif # if PY_VERSION_HEX < 0x030c0000 OK_OP(PRECALL) /* Ok, because it's used for calling. */ diff --git a/source/blender/python/intern/bpy_library_write.cc b/source/blender/python/intern/bpy_library_write.cc index d1c0b32ac07..59bf476427f 100644 --- a/source/blender/python/intern/bpy_library_write.cc +++ b/source/blender/python/intern/bpy_library_write.cc @@ -136,20 +136,25 @@ static PyObject *bpy_lib_write(BPy_PropertyRNA *self, PyObject *args, PyObject * PartialWriteContext::IDAddOperations::ADD_DEPENDENCIES | (use_fake_user ? PartialWriteContext::IDAddOperations::SET_FAKE_USER : 0))}; - Py_ssize_t pos, hash; - PyObject *key; - ID *id = nullptr; - - pos = hash = 0; - while (_PySet_NextEntry(datablocks, &pos, &key, &hash)) { - if (!pyrna_id_FromPyObject(key, &id)) { - PyErr_Format(PyExc_TypeError, "Expected an ID type, not %.200s", Py_TYPE(key)->tp_name); - return nullptr; - } - else { + if (PySet_GET_SIZE(datablocks) > 0) { + PyObject *it = PyObject_GetIter(datablocks); + PyObject *key; + while ((key = PyIter_Next(it))) { + /* Borrow from the set. */ + Py_DECREF(key); + ID *id; + if (!pyrna_id_FromPyObject(key, &id)) { + PyErr_Format(PyExc_TypeError, "Expected an ID type, not %.200s", Py_TYPE(key)->tp_name); + break; + } partial_write_ctx.id_add(id, add_options, nullptr); } + Py_DECREF(it); + if (key) { + return nullptr; + } } + BLI_assert(partial_write_ctx.is_valid()); /* write blend */ diff --git a/source/blender/python/intern/bpy_rna.cc b/source/blender/python/intern/bpy_rna.cc index eb365bf0238..a9cb115f57e 100644 --- a/source/blender/python/intern/bpy_rna.cc +++ b/source/blender/python/intern/bpy_rna.cc @@ -8639,7 +8639,7 @@ static int bpy_class_validate_recursive(PointerRNA *dummy_ptr, continue; } - /* TODO(@ideasman42): Use Python3.7x _PyObject_LookupAttr(), also in the macro below. */ + /* TODO(@ideasman42): Use #PyObject_GetOptionalAttr(), also in the macro below. */ identifier = RNA_property_identifier(prop); item = PyObject_GetAttrString(py_class, identifier); @@ -9263,7 +9263,7 @@ static PyObject *pyrna_register_class(PyObject * /*self*/, PyObject *py_class) /* Call classed register method. * Note that zero falls through, no attribute, no error. */ - switch (_PyObject_LookupAttr(py_class, bpy_intern_str_register, &py_cls_meth)) { + switch (PyObject_GetOptionalAttr(py_class, bpy_intern_str_register, &py_cls_meth)) { case 1: { PyObject *ret = PyObject_CallObject(py_cls_meth, nullptr); Py_DECREF(py_cls_meth); @@ -9378,7 +9378,7 @@ static PyObject *pyrna_unregister_class(PyObject * /*self*/, PyObject *py_class) /* Call classed unregister method. * Note that zero falls through, no attribute, no error. */ - switch (_PyObject_LookupAttr(py_class, bpy_intern_str_unregister, &py_cls_meth)) { + switch (PyObject_GetOptionalAttr(py_class, bpy_intern_str_unregister, &py_cls_meth)) { case 1: { PyObject *ret = PyObject_CallObject(py_cls_meth, nullptr); Py_DECREF(py_cls_meth);