786 lines · c
1//===-- PythonDataObjects.h--------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9//10// !! FIXME FIXME FIXME !!11//12// Python APIs nearly all can return an exception. They do this13// by returning NULL, or -1, or some such value and setting14// the exception state with PyErr_Set*(). Exceptions must be15// handled before further python API functions are called. Failure16// to do so will result in asserts on debug builds of python.17// It will also sometimes, but not usually result in crashes of18// release builds.19//20// Nearly all the code in this header does not handle python exceptions21// correctly. It should all be converted to return Expected<> or22// Error types to capture the exception.23//24// Everything in this file except functions that return Error or25// Expected<> is considered deprecated and should not be26// used in new code. If you need to use it, fix it first.27//28//29// TODOs for this file30//31// * Make all methods safe for exceptions.32//33// * Eliminate method signatures that must translate exceptions into34// empty objects or NULLs. Almost everything here should return35// Expected<>. It should be acceptable for certain operations that36// can never fail to assert instead, such as the creation of37// PythonString from a string literal.38//39// * Eliminate Reset(), and make all non-default constructors private.40// Python objects should be created with Retain<> or Take<>, and they41// should be assigned with operator=42//43// * Eliminate default constructors, make python objects always44// nonnull, and use optionals where necessary.45//46 47 48#ifndef LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_PYTHONDATAOBJECTS_H49#define LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_PYTHONDATAOBJECTS_H50 51#include "lldb/Host/Config.h"52 53#if LLDB_ENABLE_PYTHON54 55// LLDB Python header must be included first56#include "lldb-python.h"57 58#include "lldb/Host/File.h"59#include "lldb/Utility/StructuredData.h"60 61#include "llvm/ADT/ArrayRef.h"62 63namespace lldb_private {64namespace python {65 66class PythonObject;67class PythonBytes;68class PythonString;69class PythonList;70class PythonDictionary;71class PythonInteger;72class PythonException;73 74class GIL {75public:76 GIL() {77 m_state = PyGILState_Ensure();78 assert(!PyErr_Occurred());79 }80 ~GIL() { PyGILState_Release(m_state); }81 82protected:83 PyGILState_STATE m_state;84};85 86enum class PyObjectType {87 Unknown,88 None,89 Boolean,90 Integer,91 Dictionary,92 List,93 String,94 Bytes,95 ByteArray,96 Module,97 Callable,98 Tuple,99 File100};101 102enum class PyRefType {103 Borrowed, // We are not given ownership of the incoming PyObject.104 // We cannot safely hold it without calling Py_INCREF.105 Owned // We have ownership of the incoming PyObject. We should106 // not call Py_INCREF.107};108 109 110// Take a reference that you already own, and turn it into111// a PythonObject.112//113// Most python API methods will return a +1 reference114// if they succeed or NULL if and only if115// they set an exception. Use this to collect such return116// values, after checking for NULL.117//118// If T is not just PythonObject, then obj must be already be119// checked to be of the correct type.120template <typename T> T Take(PyObject *obj) {121 assert(obj);122 assert(!PyErr_Occurred());123 T thing(PyRefType::Owned, obj);124 assert(thing.IsValid());125 return thing;126}127 128// Retain a reference you have borrowed, and turn it into129// a PythonObject.130//131// A minority of python APIs return a borrowed reference132// instead of a +1. They will also return NULL if and only133// if they set an exception. Use this to collect such return134// values, after checking for NULL.135//136// If T is not just PythonObject, then obj must be already be137// checked to be of the correct type.138template <typename T> T Retain(PyObject *obj) {139 assert(obj);140 assert(!PyErr_Occurred());141 T thing(PyRefType::Borrowed, obj);142 assert(thing.IsValid());143 return thing;144}145 146// This class can be used like a utility function to convert from147// a llvm-friendly Twine into a null-terminated const char *,148// which is the form python C APIs want their strings in.149//150// Example:151// const llvm::Twine &some_twine;152// PyFoo_Bar(x, y, z, NullTerminated(some_twine));153//154// Why a class instead of a function? If the twine isn't already null155// terminated, it will need a temporary buffer to copy the string156// into. We need that buffer to stick around for the lifetime of the157// statement.158class NullTerminated {159 const char *str;160 llvm::SmallString<32> storage;161 162public:163 NullTerminated(const llvm::Twine &twine) {164 llvm::StringRef ref = twine.toNullTerminatedStringRef(storage);165 str = ref.begin();166 }167 operator const char *() { return str; }168};169 170inline llvm::Error nullDeref() {171 return llvm::createStringError(llvm::inconvertibleErrorCode(),172 "A NULL PyObject* was dereferenced");173}174 175inline llvm::Error exception(const char *s = nullptr) {176 return llvm::make_error<PythonException>(s);177}178 179inline llvm::Error keyError() {180 return llvm::createStringError(llvm::inconvertibleErrorCode(),181 "key not in dict");182}183 184inline const char *py2_const_cast(const char *s) { return s; }185 186enum class PyInitialValue { Invalid, Empty };187 188// DOC: https://docs.python.org/3/c-api/arg.html#building-values189template <typename T, typename Enable = void> struct PythonFormat;190 191template <typename T, char F> struct PassthroughFormat {192 static constexpr char format = F;193 static constexpr T get(T t) { return t; }194};195 196template <> struct PythonFormat<char *> : PassthroughFormat<char *, 's'> {};197template <>198struct PythonFormat<const char *> : PassthroughFormat<const char *, 's'> {};199template <> struct PythonFormat<char> : PassthroughFormat<char, 'b'> {};200template <>201struct PythonFormat<unsigned char> : PassthroughFormat<unsigned char, 'B'> {};202template <> struct PythonFormat<short> : PassthroughFormat<short, 'h'> {};203template <>204struct PythonFormat<unsigned short> : PassthroughFormat<unsigned short, 'H'> {};205template <> struct PythonFormat<int> : PassthroughFormat<int, 'i'> {};206template <> struct PythonFormat<bool> : PassthroughFormat<bool, 'p'> {};207template <>208struct PythonFormat<unsigned int> : PassthroughFormat<unsigned int, 'I'> {};209template <> struct PythonFormat<long> : PassthroughFormat<long, 'l'> {};210template <>211struct PythonFormat<unsigned long> : PassthroughFormat<unsigned long, 'k'> {};212template <>213struct PythonFormat<long long> : PassthroughFormat<long long, 'L'> {};214template <>215struct PythonFormat<unsigned long long>216 : PassthroughFormat<unsigned long long, 'K'> {};217template <>218struct PythonFormat<PyObject *> : PassthroughFormat<PyObject *, 'O'> {};219 220template <typename T>221struct PythonFormat<222 T, typename std::enable_if<std::is_base_of<PythonObject, T>::value>::type> {223 static constexpr char format = 'O';224 static auto get(const T &value) { return value.get(); }225};226 227class PythonObject {228public:229 PythonObject() = default;230 231 PythonObject(PyRefType type, PyObject *py_obj) {232 m_py_obj = py_obj;233 // If this is a borrowed reference, we need to convert it to234 // an owned reference by incrementing it. If it is an owned235 // reference (for example the caller allocated it with PyDict_New()236 // then we must *not* increment it.237 if (m_py_obj && Py_IsInitialized() && type == PyRefType::Borrowed)238 Py_XINCREF(m_py_obj);239 }240 241 PythonObject(const PythonObject &rhs)242 : PythonObject(PyRefType::Borrowed, rhs.m_py_obj) {}243 244 PythonObject(PythonObject &&rhs) {245 m_py_obj = rhs.m_py_obj;246 rhs.m_py_obj = nullptr;247 }248 249 ~PythonObject() { Reset(); }250 251 void Reset();252 253 void Dump(Stream &strm) const;254 255 PyObject *get() const { return m_py_obj; }256 257 PyObject *release() {258 PyObject *result = m_py_obj;259 m_py_obj = nullptr;260 return result;261 }262 263 PythonObject &operator=(PythonObject other) {264 Reset();265 m_py_obj = std::exchange(other.m_py_obj, nullptr);266 return *this;267 }268 269 PyObjectType GetObjectType() const;270 271 PythonString Repr() const;272 273 PythonString Str() const;274 275 static PythonObject ResolveNameWithDictionary(llvm::StringRef name,276 const PythonDictionary &dict);277 278 template <typename T>279 static T ResolveNameWithDictionary(llvm::StringRef name,280 const PythonDictionary &dict) {281 return ResolveNameWithDictionary(name, dict).AsType<T>();282 }283 284 PythonObject ResolveName(llvm::StringRef name) const;285 286 template <typename T> T ResolveName(llvm::StringRef name) const {287 return ResolveName(name).AsType<T>();288 }289 290 bool HasAttribute(llvm::StringRef attribute) const;291 292 PythonObject GetAttributeValue(llvm::StringRef attribute) const;293 294 bool IsNone() const { return m_py_obj == Py_None; }295 296 bool IsValid() const { return m_py_obj != nullptr; }297 298 bool IsAllocated() const { return IsValid() && !IsNone(); }299 300 explicit operator bool() const { return IsValid() && !IsNone(); }301 302 template <typename T> T AsType() const {303 if (!T::Check(m_py_obj))304 return T();305 return T(PyRefType::Borrowed, m_py_obj);306 }307 308 StructuredData::ObjectSP CreateStructuredObject() const;309 310 template <typename... T>311 llvm::Expected<PythonObject> CallMethod(const char *name,312 const T &... t) const {313 const char format[] = {'(', PythonFormat<T>::format..., ')', 0};314 PyObject *obj =315 PyObject_CallMethod(m_py_obj, py2_const_cast(name),316 py2_const_cast(format), PythonFormat<T>::get(t)...);317 if (!obj)318 return exception();319 return python::Take<PythonObject>(obj);320 }321 322 template <typename... T>323 llvm::Expected<PythonObject> Call(const T &... t) const {324 const char format[] = {'(', PythonFormat<T>::format..., ')', 0};325 PyObject *obj = PyObject_CallFunction(m_py_obj, py2_const_cast(format),326 PythonFormat<T>::get(t)...);327 if (!obj)328 return exception();329 return python::Take<PythonObject>(obj);330 }331 332 llvm::Expected<PythonObject> GetAttribute(const llvm::Twine &name) const {333 if (!m_py_obj)334 return nullDeref();335 PyObject *obj = PyObject_GetAttrString(m_py_obj, NullTerminated(name));336 if (!obj)337 return exception();338 return python::Take<PythonObject>(obj);339 }340 341 llvm::Expected<PythonObject> GetType() const {342 if (!m_py_obj)343 return nullDeref();344 PyObject *obj = PyObject_Type(m_py_obj);345 if (!obj)346 return exception();347 return python::Take<PythonObject>(obj);348 }349 350 llvm::Expected<bool> IsTrue() {351 if (!m_py_obj)352 return nullDeref();353 int r = PyObject_IsTrue(m_py_obj);354 if (r < 0)355 return exception();356 return !!r;357 }358 359 llvm::Expected<long long> AsLongLong() const;360 361 llvm::Expected<unsigned long long> AsUnsignedLongLong() const;362 363 // wraps on overflow, instead of raising an error.364 llvm::Expected<unsigned long long> AsModuloUnsignedLongLong() const;365 366 llvm::Expected<bool> IsInstance(const PythonObject &cls) {367 if (!m_py_obj || !cls.IsValid())368 return nullDeref();369 int r = PyObject_IsInstance(m_py_obj, cls.get());370 if (r < 0)371 return exception();372 return !!r;373 }374 375protected:376 PyObject *m_py_obj = nullptr;377};378 379 380// This is why C++ needs monads.381template <typename T> llvm::Expected<T> As(llvm::Expected<PythonObject> &&obj) {382 if (!obj)383 return obj.takeError();384 if (!T::Check(obj.get().get()))385 return llvm::createStringError(llvm::inconvertibleErrorCode(),386 "type error");387 return T(PyRefType::Borrowed, std::move(obj.get().get()));388}389 390template <> llvm::Expected<bool> As<bool>(llvm::Expected<PythonObject> &&obj);391 392template <>393llvm::Expected<long long> As<long long>(llvm::Expected<PythonObject> &&obj);394 395template <>396llvm::Expected<unsigned long long>397As<unsigned long long>(llvm::Expected<PythonObject> &&obj);398 399template <>400llvm::Expected<std::string> As<std::string>(llvm::Expected<PythonObject> &&obj);401 402 403template <class T> class TypedPythonObject : public PythonObject {404public:405 TypedPythonObject(PyRefType type, PyObject *py_obj) {406 if (!py_obj)407 return;408 if (T::Check(py_obj))409 PythonObject::operator=(PythonObject(type, py_obj));410 else if (type == PyRefType::Owned)411 Py_DECREF(py_obj);412 }413 414 TypedPythonObject() = default;415};416 417class PythonBytes : public TypedPythonObject<PythonBytes> {418public:419 using TypedPythonObject::TypedPythonObject;420 explicit PythonBytes(llvm::ArrayRef<uint8_t> bytes);421 PythonBytes(const uint8_t *bytes, size_t length);422 423 static bool Check(PyObject *py_obj);424 425 llvm::ArrayRef<uint8_t> GetBytes() const;426 427 size_t GetSize() const;428 429 void SetBytes(llvm::ArrayRef<uint8_t> stringbytes);430 431 StructuredData::StringSP CreateStructuredString() const;432};433 434class PythonByteArray : public TypedPythonObject<PythonByteArray> {435public:436 using TypedPythonObject::TypedPythonObject;437 explicit PythonByteArray(llvm::ArrayRef<uint8_t> bytes);438 PythonByteArray(const uint8_t *bytes, size_t length);439 PythonByteArray(const PythonBytes &object);440 441 static bool Check(PyObject *py_obj);442 443 llvm::ArrayRef<uint8_t> GetBytes() const;444 445 size_t GetSize() const;446 447 void SetBytes(llvm::ArrayRef<uint8_t> stringbytes);448 449 StructuredData::StringSP CreateStructuredString() const;450};451 452class PythonString : public TypedPythonObject<PythonString> {453public:454 using TypedPythonObject::TypedPythonObject;455 static llvm::Expected<PythonString> FromUTF8(llvm::StringRef string);456 457 PythonString() : TypedPythonObject() {} // MSVC requires this for some reason458 459 explicit PythonString(llvm::StringRef string); // safe, null on error460 461 static bool Check(PyObject *py_obj);462 463 llvm::StringRef GetString() const; // safe, empty string on error464 465 llvm::Expected<llvm::StringRef> AsUTF8() const;466 467 size_t GetSize() const;468 469 void SetString(llvm::StringRef string); // safe, null on error470 471 StructuredData::StringSP CreateStructuredString() const;472};473 474class PythonInteger : public TypedPythonObject<PythonInteger> {475public:476 using TypedPythonObject::TypedPythonObject;477 478 PythonInteger() : TypedPythonObject() {} // MSVC requires this for some reason479 480 explicit PythonInteger(int64_t value);481 482 static bool Check(PyObject *py_obj);483 484 void SetInteger(int64_t value);485 486 StructuredData::IntegerSP CreateStructuredInteger() const;487 488 StructuredData::UnsignedIntegerSP CreateStructuredUnsignedInteger() const;489 490 StructuredData::SignedIntegerSP CreateStructuredSignedInteger() const;491};492 493class PythonBoolean : public TypedPythonObject<PythonBoolean> {494public:495 using TypedPythonObject::TypedPythonObject;496 497 explicit PythonBoolean(bool value);498 499 static bool Check(PyObject *py_obj);500 501 bool GetValue() const;502 503 void SetValue(bool value);504 505 StructuredData::BooleanSP CreateStructuredBoolean() const;506};507 508class PythonList : public TypedPythonObject<PythonList> {509public:510 using TypedPythonObject::TypedPythonObject;511 512 PythonList() : TypedPythonObject() {} // MSVC requires this for some reason513 514 explicit PythonList(PyInitialValue value);515 explicit PythonList(int list_size);516 517 static bool Check(PyObject *py_obj);518 519 uint32_t GetSize() const;520 521 PythonObject GetItemAtIndex(uint32_t index) const;522 523 void SetItemAtIndex(uint32_t index, const PythonObject &object);524 525 void AppendItem(const PythonObject &object);526 527 StructuredData::ArraySP CreateStructuredArray() const;528};529 530class PythonTuple : public TypedPythonObject<PythonTuple> {531public:532 using TypedPythonObject::TypedPythonObject;533 534 explicit PythonTuple(PyInitialValue value);535 explicit PythonTuple(int tuple_size);536 PythonTuple(std::initializer_list<PythonObject> objects);537 PythonTuple(std::initializer_list<PyObject *> objects);538 539 static bool Check(PyObject *py_obj);540 541 uint32_t GetSize() const;542 543 PythonObject GetItemAtIndex(uint32_t index) const;544 545 void SetItemAtIndex(uint32_t index, const PythonObject &object);546 547 StructuredData::ArraySP CreateStructuredArray() const;548};549 550class PythonDictionary : public TypedPythonObject<PythonDictionary> {551public:552 using TypedPythonObject::TypedPythonObject;553 554 PythonDictionary() : TypedPythonObject() {} // MSVC requires this for some reason555 556 explicit PythonDictionary(PyInitialValue value);557 558 static bool Check(PyObject *py_obj);559 560 bool HasKey(const llvm::Twine &key) const;561 562 uint32_t GetSize() const;563 564 PythonList GetKeys() const;565 566 PythonObject GetItemForKey(const PythonObject &key) const; // DEPRECATED567 void SetItemForKey(const PythonObject &key,568 const PythonObject &value); // DEPRECATED569 570 llvm::Expected<PythonObject> GetItem(const PythonObject &key) const;571 llvm::Expected<PythonObject> GetItem(const llvm::Twine &key) const;572 llvm::Error SetItem(const PythonObject &key, const PythonObject &value) const;573 llvm::Error SetItem(const llvm::Twine &key, const PythonObject &value) const;574 575 StructuredData::DictionarySP CreateStructuredDictionary() const;576};577 578class PythonModule : public TypedPythonObject<PythonModule> {579public:580 using TypedPythonObject::TypedPythonObject;581 582 static bool Check(PyObject *py_obj);583 584 static PythonModule BuiltinsModule();585 586 static PythonModule MainModule();587 588 static PythonModule AddModule(llvm::StringRef module);589 590 // safe, returns invalid on error;591 static PythonModule ImportModule(llvm::StringRef name) {592 std::string s = std::string(name);593 auto mod = Import(s.c_str());594 if (!mod) {595 llvm::consumeError(mod.takeError());596 return PythonModule();597 }598 return std::move(mod.get());599 }600 601 static llvm::Expected<PythonModule> Import(const llvm::Twine &name);602 603 llvm::Expected<PythonObject> Get(const llvm::Twine &name);604 605 PythonDictionary GetDictionary() const;606};607 608class PythonCallable : public TypedPythonObject<PythonCallable> {609public:610 using TypedPythonObject::TypedPythonObject;611 612 struct ArgInfo {613 /* the largest number of positional arguments this callable614 * can accept, or UNBOUNDED, ie UINT_MAX if it's a varargs615 * function and can accept an arbitrary number */616 unsigned max_positional_args;617 static constexpr unsigned UNBOUNDED = UINT_MAX; // FIXME c++17 inline618 };619 620 static bool Check(PyObject *py_obj);621 622 llvm::Expected<ArgInfo> GetArgInfo() const;623 624 PythonObject operator()();625 626 PythonObject operator()(std::initializer_list<PyObject *> args);627 628 PythonObject operator()(std::initializer_list<PythonObject> args);629 630 template <typename Arg, typename... Args>631 PythonObject operator()(const Arg &arg, Args... args) {632 return operator()({arg, args...});633 }634};635 636class PythonFile : public TypedPythonObject<PythonFile> {637public:638 using TypedPythonObject::TypedPythonObject;639 640 PythonFile() : TypedPythonObject() {} // MSVC requires this for some reason641 642 static bool Check(PyObject *py_obj);643 644 static llvm::Expected<PythonFile> FromFile(File &file,645 const char *mode = nullptr);646 647 llvm::Expected<lldb::FileSP> ConvertToFile(bool borrowed = false);648 llvm::Expected<lldb::FileSP>649 ConvertToFileForcingUseOfScriptingIOMethods(bool borrowed = false);650};651 652class PythonException : public llvm::ErrorInfo<PythonException> {653private:654 PyObject *m_exception_type, *m_exception, *m_traceback;655 PyObject *m_repr_bytes;656 657public:658 static char ID;659 const char *toCString() const;660 PythonException(const char *caller = nullptr);661 void Restore();662 ~PythonException() override;663 void log(llvm::raw_ostream &OS) const override;664 std::error_code convertToErrorCode() const override;665 bool Matches(PyObject *exc) const;666 std::string ReadBacktrace() const;667};668 669// This extracts the underlying T out of an Expected<T> and returns it.670// If the Expected is an Error instead of a T, that error will be converted671// into a python exception, and this will return a default-constructed T.672//673// This is appropriate for use right at the boundary of python calling into674// C++, such as in a SWIG typemap. In such a context you should simply675// check if the returned T is valid, and if it is, return a NULL back676// to python. This will result in the Error being raised as an exception677// from python code's point of view.678//679// For example:680// ```681// Expected<Foo *> efoop = some_cpp_function();682// Foo *foop = unwrapOrSetPythonException(efoop);683// if (!foop)684// return NULL;685// do_something(*foop);686//687// If the Error returned was itself created because a python exception was688// raised when C++ code called into python, then the original exception689// will be restored. Otherwise a simple string exception will be raised.690template <typename T> T unwrapOrSetPythonException(llvm::Expected<T> expected) {691 if (expected)692 return expected.get();693 llvm::handleAllErrors(694 expected.takeError(), [](PythonException &E) { E.Restore(); },695 [](const llvm::ErrorInfoBase &E) {696 PyErr_SetString(PyExc_Exception, E.message().c_str());697 });698 return T();699}700 701// This is only here to help incrementally migrate old, exception-unsafe702// code.703template <typename T> T unwrapIgnoringErrors(llvm::Expected<T> expected) {704 if (expected)705 return std::move(expected.get());706 llvm::consumeError(expected.takeError());707 return T();708}709 710llvm::Expected<PythonObject> runStringOneLine(const llvm::Twine &string,711 const PythonDictionary &globals,712 const PythonDictionary &locals);713 714llvm::Expected<PythonObject> runStringMultiLine(const llvm::Twine &string,715 const PythonDictionary &globals,716 const PythonDictionary &locals);717 718// Sometimes the best way to interact with a python interpreter is719// to run some python code. You construct a PythonScript with720// script string. The script assigns some function to `_function_`721// and you get a C++ callable object that calls the python function.722//723// Example:724//725// const char script[] = R"(726// def main(x, y):727// ....728// )";729//730// Expected<PythonObject> cpp_foo_wrapper(PythonObject x, PythonObject y) {731// // no need to synchronize access to this global, we already have the GIL732// static PythonScript foo(script)733// return foo(x, y);734// }735class PythonScript {736 const char *script;737 PythonCallable function;738 739 llvm::Error Init();740 741public:742 PythonScript(const char *script) : script(script), function() {}743 744 template <typename... Args>745 llvm::Expected<PythonObject> operator()(Args &&... args) {746 if (llvm::Error error = Init())747 return std::move(error);748 return function.Call(std::forward<Args>(args)...);749 }750};751 752class StructuredPythonObject : public StructuredData::Generic {753public:754 StructuredPythonObject() : StructuredData::Generic() {}755 756 // Take ownership of the object we received.757 StructuredPythonObject(PythonObject obj)758 : StructuredData::Generic(obj.release()) {}759 760 ~StructuredPythonObject() override {761 // Hand ownership back to a (temporary) PythonObject instance and let it762 // take care of releasing it.763 PythonObject(PyRefType::Owned, static_cast<PyObject *>(GetValue()));764 }765 766 bool IsValid() const override { return GetValue() && GetValue() != Py_None; }767 768 void Serialize(llvm::json::OStream &s) const override;769 770private:771 StructuredPythonObject(const StructuredPythonObject &) = delete;772 const StructuredPythonObject &773 operator=(const StructuredPythonObject &) = delete;774};775 776PyObject *RunString(const char *str, int start, PyObject *globals,777 PyObject *locals);778int RunSimpleString(const char *str);779 780} // namespace python781} // namespace lldb_private782 783#endif784 785#endif // LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_PYTHONDATAOBJECTS_H786