diff --git a/gensim/models/_fasttext_bin.py b/gensim/models/_fasttext_bin.py index 31d85c5074..ab88dbffda 100644 --- a/gensim/models/_fasttext_bin.py +++ b/gensim/models/_fasttext_bin.py @@ -79,6 +79,49 @@ def _yield_field_names(): _FIELD_NAMES = sorted(set(_yield_field_names())) Model = collections.namedtuple('Model', _FIELD_NAMES) +"""Holds data loaded from the Facebook binary. + +Fields +------ +dim : int + The dimensionality of the vectors. +ws : int + The window size. +epoch : int + The number of training epochs. +neg : int + If non-zero, indicates that the model uses negative sampling. +loss : int + If equal to 1, indicates that the model uses hierarchical sampling. +model : int + If equal to 2, indicates that the model uses skip-grams. +bucket : int + The number of buckets. +min_count : int + The threshold below which the model ignores terms. +t : float + The sample threshold. +minn : int + The minimum ngram length. +maxn : int + The maximum ngram length. +raw_vocab : collections.OrderedDict + A map from words (str) to their frequency (int). The order in the dict + corresponds to the order of the words in the Facebook binary. +nwords : int + The number of words. +vocab_size : int + The size of the vocabulary. +vectors_ngrams : numpy.array + This is a matrix that contains vectors learned by the model. + Each row corresponds to a vector. + The number of vectors is equal to the number of words plus the number of buckets. + The number of columns is equal to the vector dimensionality. +hidden_output : numpy.array + This is a matrix that contains the shallow neural network output. + This array has the same dimensions as vectors_ngrams. + May be None - in that case, it is impossible to continue training the model. +""" def _struct_unpack(fin, fmt): @@ -177,7 +220,7 @@ def _load_matrix(fin, new_format=True): return matrix -def load(fin, encoding='utf-8'): +def load(fin, encoding='utf-8', full_model=True): """Load a model from a binary stream. Parameters @@ -186,6 +229,9 @@ def load(fin, encoding='utf-8'): The readable binary stream. encoding : str, optional The encoding to use for decoding text + full_model : boolean, optional + If False, skips loading the hidden output matrix. This saves a fair bit + of CPU time and RAM, but prevents training continuation. Returns ------- @@ -209,10 +255,12 @@ def load(fin, encoding='utf-8'): vectors_ngrams = _load_matrix(fin, new_format=new_format) - hidden_output = _load_matrix(fin, new_format=new_format) - model.update(vectors_ngrams=vectors_ngrams, hidden_output=hidden_output) - - assert fin.read() == b'', 'expected to reach EOF' + if not full_model: + hidden_output = None + else: + hidden_output = _load_matrix(fin, new_format=new_format) + assert fin.read() == b'', 'expected to reach EOF' + model.update(vectors_ngrams=vectors_ngrams, hidden_output=hidden_output) model = {k: v for k, v in model.items() if k in _FIELD_NAMES} return Model(**model) diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index c42dd6ef33..1e21639afa 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -608,6 +608,7 @@ static CYTHON_INLINE float __PYX_NAN() { #define __PYX_HAVE__gensim__models___utils_any2vec #define __PYX_HAVE_API__gensim__models___utils_any2vec /* Early includes */ +#include "stdint_wrapper.h" #include #include #include "numpy/arrayobject.h" @@ -843,11 +844,10 @@ static const char *__pyx_filename; static const char *__pyx_f[] = { "gensim/models/_utils_any2vec.pyx", "__init__.pxd", - "stringsource", "type.pxd", }; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":776 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":776 * # in Cython to enable them only on the right systems. * * ctypedef npy_int8 int8_t # <<<<<<<<<<<<<< @@ -856,7 +856,7 @@ static const char *__pyx_f[] = { */ typedef npy_int8 __pyx_t_5numpy_int8_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":777 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":777 * * ctypedef npy_int8 int8_t * ctypedef npy_int16 int16_t # <<<<<<<<<<<<<< @@ -865,7 +865,7 @@ typedef npy_int8 __pyx_t_5numpy_int8_t; */ typedef npy_int16 __pyx_t_5numpy_int16_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":778 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":778 * ctypedef npy_int8 int8_t * ctypedef npy_int16 int16_t * ctypedef npy_int32 int32_t # <<<<<<<<<<<<<< @@ -874,7 +874,7 @@ typedef npy_int16 __pyx_t_5numpy_int16_t; */ typedef npy_int32 __pyx_t_5numpy_int32_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":779 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":779 * ctypedef npy_int16 int16_t * ctypedef npy_int32 int32_t * ctypedef npy_int64 int64_t # <<<<<<<<<<<<<< @@ -883,7 +883,7 @@ typedef npy_int32 __pyx_t_5numpy_int32_t; */ typedef npy_int64 __pyx_t_5numpy_int64_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":783 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":783 * #ctypedef npy_int128 int128_t * * ctypedef npy_uint8 uint8_t # <<<<<<<<<<<<<< @@ -892,7 +892,7 @@ typedef npy_int64 __pyx_t_5numpy_int64_t; */ typedef npy_uint8 __pyx_t_5numpy_uint8_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":784 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":784 * * ctypedef npy_uint8 uint8_t * ctypedef npy_uint16 uint16_t # <<<<<<<<<<<<<< @@ -901,7 +901,7 @@ typedef npy_uint8 __pyx_t_5numpy_uint8_t; */ typedef npy_uint16 __pyx_t_5numpy_uint16_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":785 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":785 * ctypedef npy_uint8 uint8_t * ctypedef npy_uint16 uint16_t * ctypedef npy_uint32 uint32_t # <<<<<<<<<<<<<< @@ -910,7 +910,7 @@ typedef npy_uint16 __pyx_t_5numpy_uint16_t; */ typedef npy_uint32 __pyx_t_5numpy_uint32_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":786 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":786 * ctypedef npy_uint16 uint16_t * ctypedef npy_uint32 uint32_t * ctypedef npy_uint64 uint64_t # <<<<<<<<<<<<<< @@ -919,7 +919,7 @@ typedef npy_uint32 __pyx_t_5numpy_uint32_t; */ typedef npy_uint64 __pyx_t_5numpy_uint64_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":790 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":790 * #ctypedef npy_uint128 uint128_t * * ctypedef npy_float32 float32_t # <<<<<<<<<<<<<< @@ -928,7 +928,7 @@ typedef npy_uint64 __pyx_t_5numpy_uint64_t; */ typedef npy_float32 __pyx_t_5numpy_float32_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":791 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":791 * * ctypedef npy_float32 float32_t * ctypedef npy_float64 float64_t # <<<<<<<<<<<<<< @@ -937,7 +937,7 @@ typedef npy_float32 __pyx_t_5numpy_float32_t; */ typedef npy_float64 __pyx_t_5numpy_float64_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":800 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":800 * # The int types are mapped a bit surprising -- * # numpy.int corresponds to 'l' and numpy.long to 'q' * ctypedef npy_long int_t # <<<<<<<<<<<<<< @@ -946,7 +946,7 @@ typedef npy_float64 __pyx_t_5numpy_float64_t; */ typedef npy_long __pyx_t_5numpy_int_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":801 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":801 * # numpy.int corresponds to 'l' and numpy.long to 'q' * ctypedef npy_long int_t * ctypedef npy_longlong long_t # <<<<<<<<<<<<<< @@ -955,7 +955,7 @@ typedef npy_long __pyx_t_5numpy_int_t; */ typedef npy_longlong __pyx_t_5numpy_long_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":802 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":802 * ctypedef npy_long int_t * ctypedef npy_longlong long_t * ctypedef npy_longlong longlong_t # <<<<<<<<<<<<<< @@ -964,7 +964,7 @@ typedef npy_longlong __pyx_t_5numpy_long_t; */ typedef npy_longlong __pyx_t_5numpy_longlong_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":804 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":804 * ctypedef npy_longlong longlong_t * * ctypedef npy_ulong uint_t # <<<<<<<<<<<<<< @@ -973,7 +973,7 @@ typedef npy_longlong __pyx_t_5numpy_longlong_t; */ typedef npy_ulong __pyx_t_5numpy_uint_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":805 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":805 * * ctypedef npy_ulong uint_t * ctypedef npy_ulonglong ulong_t # <<<<<<<<<<<<<< @@ -982,7 +982,7 @@ typedef npy_ulong __pyx_t_5numpy_uint_t; */ typedef npy_ulonglong __pyx_t_5numpy_ulong_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":806 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":806 * ctypedef npy_ulong uint_t * ctypedef npy_ulonglong ulong_t * ctypedef npy_ulonglong ulonglong_t # <<<<<<<<<<<<<< @@ -991,7 +991,7 @@ typedef npy_ulonglong __pyx_t_5numpy_ulong_t; */ typedef npy_ulonglong __pyx_t_5numpy_ulonglong_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":808 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":808 * ctypedef npy_ulonglong ulonglong_t * * ctypedef npy_intp intp_t # <<<<<<<<<<<<<< @@ -1000,7 +1000,7 @@ typedef npy_ulonglong __pyx_t_5numpy_ulonglong_t; */ typedef npy_intp __pyx_t_5numpy_intp_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":809 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":809 * * ctypedef npy_intp intp_t * ctypedef npy_uintp uintp_t # <<<<<<<<<<<<<< @@ -1009,7 +1009,7 @@ typedef npy_intp __pyx_t_5numpy_intp_t; */ typedef npy_uintp __pyx_t_5numpy_uintp_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":811 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":811 * ctypedef npy_uintp uintp_t * * ctypedef npy_double float_t # <<<<<<<<<<<<<< @@ -1018,7 +1018,7 @@ typedef npy_uintp __pyx_t_5numpy_uintp_t; */ typedef npy_double __pyx_t_5numpy_float_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":812 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":812 * * ctypedef npy_double float_t * ctypedef npy_double double_t # <<<<<<<<<<<<<< @@ -1027,7 +1027,7 @@ typedef npy_double __pyx_t_5numpy_float_t; */ typedef npy_double __pyx_t_5numpy_double_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":813 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":813 * ctypedef npy_double float_t * ctypedef npy_double double_t * ctypedef npy_longdouble longdouble_t # <<<<<<<<<<<<<< @@ -1061,9 +1061,8 @@ static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(do /*--- Type declarations ---*/ -struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":815 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":815 * ctypedef npy_longdouble longdouble_t * * ctypedef npy_cfloat cfloat_t # <<<<<<<<<<<<<< @@ -1072,7 +1071,7 @@ struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; */ typedef npy_cfloat __pyx_t_5numpy_cfloat_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":816 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":816 * * ctypedef npy_cfloat cfloat_t * ctypedef npy_cdouble cdouble_t # <<<<<<<<<<<<<< @@ -1081,7 +1080,7 @@ typedef npy_cfloat __pyx_t_5numpy_cfloat_t; */ typedef npy_cdouble __pyx_t_5numpy_cdouble_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":817 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":817 * ctypedef npy_cfloat cfloat_t * ctypedef npy_cdouble cdouble_t * ctypedef npy_clongdouble clongdouble_t # <<<<<<<<<<<<<< @@ -1090,7 +1089,7 @@ typedef npy_cdouble __pyx_t_5numpy_cdouble_t; */ typedef npy_clongdouble __pyx_t_5numpy_clongdouble_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":819 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":819 * ctypedef npy_clongdouble clongdouble_t * * ctypedef npy_cdouble complex_t # <<<<<<<<<<<<<< @@ -1099,19 +1098,6 @@ typedef npy_clongdouble __pyx_t_5numpy_clongdouble_t; */ typedef npy_cdouble __pyx_t_5numpy_complex_t; -/* "cfunc.to_py":64 - * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" - */ -struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py { - PyObject_HEAD - PyObject *(*__pyx_v_f)(PyObject *); -}; - - /* --- Runtime support code (head) --- */ /* Refnanny.proto */ #ifndef CYTHON_REFNANNY @@ -1186,6 +1172,16 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject /* GetBuiltinName.proto */ static PyObject *__Pyx_GetBuiltinName(PyObject *name); +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely((Py_TYPE(obj) == type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* unicode_iter.proto */ +static CYTHON_INLINE int __Pyx_init_unicode_iteration( + PyObject* ustring, Py_ssize_t *length, void** data, int *kind); + /* UnicodeAsUCS4.proto */ static CYTHON_INLINE Py_UCS4 __Pyx_PyUnicode_AsPy_UCS4(PyObject*); @@ -1198,85 +1194,6 @@ static CYTHON_INLINE Py_UCS4 __Pyx_PyUnicode_AsPy_UCS4(PyObject*); #endif static long __Pyx__PyObject_Ord(PyObject* c); -/* GetModuleGlobalName.proto */ -#if CYTHON_USE_DICT_VERSIONS -#define __Pyx_GetModuleGlobalName(var, name) {\ - static PY_UINT64_T __pyx_dict_version = 0;\ - static PyObject *__pyx_dict_cached_value = NULL;\ - (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_d))) ?\ - (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ - __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -#define __Pyx_GetModuleGlobalNameUncached(var, name) {\ - PY_UINT64_T __pyx_dict_version;\ - PyObject *__pyx_dict_cached_value;\ - (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); -#else -#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) -#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); -#endif - -/* PyCFunctionFastCall.proto */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - -/* PyFunctionFastCall.proto */ -#if CYTHON_FAST_PYCALL -#define __Pyx_PyFunction_FastCall(func, args, nargs)\ - __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) -#endif -#define __Pyx_BUILD_ASSERT_EXPR(cond)\ - (sizeof(char [1 - 2*!(cond)]) - 1) -#ifndef Py_MEMBER_SIZE -#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) -#endif - static size_t __pyx_pyframe_localsplus_offset = 0; - #include "frameobject.h" - #define __Pxy_PyFrame_Initialize_Offsets()\ - ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ - (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) - #define __Pyx_PyFrame_GetLocalsplus(frame)\ - (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) -#endif - -/* PyObjectCall.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); -#else -#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) -#endif - -/* PyObjectCall2Args.proto */ -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); - -/* PyObjectCallMethO.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); -#endif - -/* PyObjectCallOneArg.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); - -/* ArgTypeTest.proto */ -#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ - ((likely((Py_TYPE(obj) == type) | (none_allowed && (obj == Py_None)))) ? 1 :\ - __Pyx__ArgTypeTest(obj, type, name, exact)) -static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); - -/* unicode_iter.proto */ -static CYTHON_INLINE int __Pyx_init_unicode_iteration( - PyObject* ustring, Py_ssize_t *length, void** data, int *kind); - /* PyObjectFormatSimple.proto */ #if CYTHON_COMPILING_IN_PYPY #define __Pyx_PyObject_FormatSimple(s, f) (\ @@ -1306,6 +1223,13 @@ static CYTHON_INLINE int __Pyx_init_unicode_iteration( static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_count, Py_ssize_t result_ulength, Py_UCS4 max_char); +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + /* PyIntBinop.proto */ #if !CYTHON_COMPILING_IN_PYPY static PyObject* __Pyx_PyInt_AddObjC(PyObject *op1, PyObject *op2, long intval, int inplace); @@ -1347,6 +1271,47 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ const char* function_name); +/* PyCFunctionFastCall.proto */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); +#else +#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); +#else +#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif + +/* PyObjectCall2Args.proto */ +static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectCallOneArg.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + /* PyThreadStateGet.proto */ #if CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; @@ -1441,70 +1406,6 @@ static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); #endif -/* FetchCommonType.proto */ -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); - -/* CythonFunction.proto */ -#define __Pyx_CyFunction_USED 1 -#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 -#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 -#define __Pyx_CYFUNCTION_CCLASS 0x04 -#define __Pyx_CyFunction_GetClosure(f)\ - (((__pyx_CyFunctionObject *) (f))->func_closure) -#define __Pyx_CyFunction_GetClassObj(f)\ - (((__pyx_CyFunctionObject *) (f))->func_classobj) -#define __Pyx_CyFunction_Defaults(type, f)\ - ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) -#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ - ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) -typedef struct { - PyCFunctionObject func; -#if PY_VERSION_HEX < 0x030500A0 - PyObject *func_weakreflist; -#endif - PyObject *func_dict; - PyObject *func_name; - PyObject *func_qualname; - PyObject *func_doc; - PyObject *func_globals; - PyObject *func_code; - PyObject *func_closure; - PyObject *func_classobj; - void *defaults; - int defaults_pyobjects; - int flags; - PyObject *defaults_tuple; - PyObject *defaults_kwdict; - PyObject *(*defaults_getter)(PyObject *); - PyObject *func_annotations; -} __pyx_CyFunctionObject; -static PyTypeObject *__pyx_CyFunctionType = 0; -#define __Pyx_CyFunction_Check(obj) (__Pyx_TypeCheck(obj, __pyx_CyFunctionType)) -#define __Pyx_CyFunction_NewEx(ml, flags, qualname, self, module, globals, code)\ - __Pyx_CyFunction_New(__pyx_CyFunctionType, ml, flags, qualname, self, module, globals, code) -static PyObject *__Pyx_CyFunction_New(PyTypeObject *, PyMethodDef *ml, - int flags, PyObject* qualname, - PyObject *self, - PyObject *module, PyObject *globals, - PyObject* code); -static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *m, - size_t size, - int pyobjects); -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, - PyObject *tuple); -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, - PyObject *dict); -static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, - PyObject *dict); -static int __pyx_CyFunction_init(void); - -/* PyObject_GenericGetAttrNoDict.proto */ -#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 -static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name); -#else -#define __Pyx_PyObject_GenericGetAttrNoDict PyObject_GenericGetAttr -#endif - /* TypeImport.proto */ #ifndef __PYX_HAVE_RT_ImportType_proto #define __PYX_HAVE_RT_ImportType_proto @@ -1549,7 +1450,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename); /* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value); /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value); @@ -1661,9 +1562,18 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__NPY_TYPES(enum NPY_TYPES v /* CIntFromPy.proto */ static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *); +/* CIntFromPy.proto */ +static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); + /* CIntFromPy.proto */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); + /* CIntFromPy.proto */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); @@ -1717,13 +1627,12 @@ static PyTypeObject *__pyx_ptype_5numpy_ufunc = 0; static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *, char *, char *, int *); /*proto*/ /* Module declarations from 'gensim.models._utils_any2vec' */ -static PyTypeObject *__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObject *); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObject *); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *, int __pyx_skip_dispatch); /*proto*/ +static unsigned char __pyx_v_6gensim_6models_14_utils_any2vec__MB_MASK; +static unsigned char __pyx_v_6gensim_6models_14_utils_any2vec__MB_START; +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ -static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*)(PyObject *)); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ #define __Pyx_MODULE_NAME "gensim.models._utils_any2vec" extern int __pyx_module_is_main_gensim__models___utils_any2vec; int __pyx_module_is_main_gensim__models___utils_any2vec = 0; @@ -1734,32 +1643,27 @@ static PyObject *__pyx_builtin_ValueError; static PyObject *__pyx_builtin_RuntimeError; static PyObject *__pyx_builtin_ImportError; static const char __pyx_k_[] = "<"; -static const char __pyx_k_b[] = "b"; +static const char __pyx_k_s[] = "<%s>"; static const char __pyx_k__2[] = ">"; static const char __pyx_k_np[] = "np"; static const char __pyx_k_PY2[] = "PY2"; static const char __pyx_k_six[] = "six"; -static const char __pyx_k_int8[] = "int8"; static const char __pyx_k_main[] = "__main__"; static const char __pyx_k_name[] = "__name__"; static const char __pyx_k_test[] = "__test__"; static const char __pyx_k_word[] = "word"; -static const char __pyx_k_wrap[] = "wrap"; static const char __pyx_k_max_n[] = "max_n"; static const char __pyx_k_min_n[] = "min_n"; static const char __pyx_k_numpy[] = "numpy"; static const char __pyx_k_range[] = "range"; +static const char __pyx_k_utf_8[] = "utf-8"; +static const char __pyx_k_encode[] = "encode"; static const char __pyx_k_import[] = "__import__"; -static const char __pyx_k_uint32[] = "uint32"; static const char __pyx_k_ValueError[] = "ValueError"; static const char __pyx_k_ImportError[] = "ImportError"; -static const char __pyx_k_byte_to_int[] = "_byte_to_int"; -static const char __pyx_k_cfunc_to_py[] = "cfunc.to_py"; static const char __pyx_k_RuntimeError[] = "RuntimeError"; -static const char __pyx_k_stringsource[] = "stringsource"; static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; static const char __pyx_k_ndarray_is_not_C_contiguous[] = "ndarray is not C contiguous"; -static const char __pyx_k_Pyx_CFunc_object____object___t[] = "__Pyx_CFunc_object____object___to_py..wrap"; static const char __pyx_k_numpy_core_multiarray_failed_to[] = "numpy.core.multiarray failed to import"; static const char __pyx_k_unknown_dtype_code_in_numpy_pxd[] = "unknown dtype code in numpy.pxd (%d)"; static const char __pyx_k_Format_string_allocated_too_shor[] = "Format string allocated too short, see comment in numpy.pxd"; @@ -1774,16 +1678,12 @@ static PyObject *__pyx_kp_u_Format_string_allocated_too_shor_2; static PyObject *__pyx_n_s_ImportError; static PyObject *__pyx_kp_u_Non_native_byte_order_not_suppor; static PyObject *__pyx_n_s_PY2; -static PyObject *__pyx_n_s_Pyx_CFunc_object____object___t; static PyObject *__pyx_n_s_RuntimeError; static PyObject *__pyx_n_s_ValueError; static PyObject *__pyx_kp_u__2; -static PyObject *__pyx_n_s_b; -static PyObject *__pyx_n_s_byte_to_int; -static PyObject *__pyx_n_s_cfunc_to_py; static PyObject *__pyx_n_s_cline_in_traceback; +static PyObject *__pyx_n_s_encode; static PyObject *__pyx_n_s_import; -static PyObject *__pyx_n_s_int8; static PyObject *__pyx_n_s_main; static PyObject *__pyx_n_s_max_n; static PyObject *__pyx_n_s_min_n; @@ -1795,23 +1695,20 @@ static PyObject *__pyx_n_s_numpy; static PyObject *__pyx_kp_s_numpy_core_multiarray_failed_to; static PyObject *__pyx_kp_s_numpy_core_umath_failed_to_impor; static PyObject *__pyx_n_s_range; +static PyObject *__pyx_kp_s_s; static PyObject *__pyx_n_s_six; -static PyObject *__pyx_kp_s_stringsource; static PyObject *__pyx_n_s_test; -static PyObject *__pyx_n_s_uint32; static PyObject *__pyx_kp_u_unknown_dtype_code_in_numpy_pxd; +static PyObject *__pyx_kp_s_utf_8; static PyObject *__pyx_n_s_word; -static PyObject *__pyx_n_s_wrap; -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info); /* proto */ -static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /* proto */ -static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_int_0; static PyObject *__pyx_int_1; -static PyObject *__pyx_int_16777619; static PyObject *__pyx_tuple__3; static PyObject *__pyx_tuple__4; static PyObject *__pyx_tuple__5; @@ -1819,465 +1716,198 @@ static PyObject *__pyx_tuple__6; static PyObject *__pyx_tuple__7; static PyObject *__pyx_tuple__8; static PyObject *__pyx_tuple__9; -static PyObject *__pyx_tuple__10; -static PyObject *__pyx_codeobj__11; /* Late includes */ -/* "gensim/models/_utils_any2vec.pyx":15 - * +/* "gensim/models/_utils_any2vec.pyx":22 * - * cdef _byte_to_int_py3(b): # <<<<<<<<<<<<<< - * return b * + * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< + * """Calculate hash based on `bytez`. + * Reproduce `hash method from Facebook fastText implementation */ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObject *__pyx_v_b) { +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject *__pyx_v_bytez, CYTHON_UNUSED int __pyx_skip_dispatch) { + uint32_t __pyx_v_h; + char __pyx_v_b; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("_byte_to_int_py3", 0); + PyObject *__pyx_t_1 = NULL; + char *__pyx_t_2; + char *__pyx_t_3; + char *__pyx_t_4; + char *__pyx_t_5; + PyObject *__pyx_t_6 = NULL; + __Pyx_RefNannySetupContext("ft_hash_bytes", 0); + + /* "gensim/models/_utils_any2vec.pyx":38 + * + * """ + * cdef uint32_t h = 2166136261 # <<<<<<<<<<<<<< + * cdef char b + * + */ + __pyx_v_h = 0x811C9DC5; + + /* "gensim/models/_utils_any2vec.pyx":41 + * cdef char b + * + * for b in bytez: # <<<<<<<<<<<<<< + * h = h ^ (b) + * h = h * 16777619 + */ + if (unlikely(__pyx_v_bytez == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(0, 41, __pyx_L1_error) + } + __Pyx_INCREF(__pyx_v_bytez); + __pyx_t_1 = __pyx_v_bytez; + __pyx_t_3 = PyBytes_AS_STRING(__pyx_t_1); + __pyx_t_4 = (__pyx_t_3 + PyBytes_GET_SIZE(__pyx_t_1)); + for (__pyx_t_5 = __pyx_t_3; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { + __pyx_t_2 = __pyx_t_5; + __pyx_v_b = (__pyx_t_2[0]); + + /* "gensim/models/_utils_any2vec.pyx":42 + * + * for b in bytez: + * h = h ^ (b) # <<<<<<<<<<<<<< + * h = h * 16777619 + * return h + */ + __pyx_v_h = (__pyx_v_h ^ ((uint32_t)((int8_t)__pyx_v_b))); + + /* "gensim/models/_utils_any2vec.pyx":43 + * for b in bytez: + * h = h ^ (b) + * h = h * 16777619 # <<<<<<<<<<<<<< + * return h + * + */ + __pyx_v_h = (__pyx_v_h * 0x1000193); + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":16 + /* "gensim/models/_utils_any2vec.pyx":44 + * h = h ^ (b) + * h = h * 16777619 + * return h # <<<<<<<<<<<<<< * - * cdef _byte_to_int_py3(b): - * return b # <<<<<<<<<<<<<< * - * cdef _byte_to_int_py2(b): */ __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_b); - __pyx_r = __pyx_v_b; + __pyx_t_6 = __Pyx_PyInt_From_uint32_t(__pyx_v_h); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 44, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":15 - * + /* "gensim/models/_utils_any2vec.pyx":22 * - * cdef _byte_to_int_py3(b): # <<<<<<<<<<<<<< - * return b * + * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< + * """Calculate hash based on `bytez`. + * Reproduce `hash method from Facebook fastText implementation */ /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("gensim.models._utils_any2vec.ft_hash_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":18 - * return b - * - * cdef _byte_to_int_py2(b): # <<<<<<<<<<<<<< - * return ord(b) - * - */ +/* Python wrapper */ +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez); /*proto*/ +static char __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash_bytes[] = "ft_hash_bytes(bytes bytez)\nCalculate hash based on `bytez`.\n Reproduce `hash method from Facebook fastText implementation\n `_.\n\n Parameters\n ----------\n bytez : bytes\n The string whose hash needs to be calculated, encoded as UTF-8.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("ft_hash_bytes (wrapper)", 0); + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bytez), (&PyBytes_Type), 1, "bytez", 1))) __PYX_ERR(0, 22, __pyx_L1_error) + __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_self, ((PyObject*)__pyx_v_bytez)); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObject *__pyx_v_b) { +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - long __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannySetupContext("_byte_to_int_py2", 0); - - /* "gensim/models/_utils_any2vec.pyx":19 - * - * cdef _byte_to_int_py2(b): - * return ord(b) # <<<<<<<<<<<<<< - * - * _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 - */ + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("ft_hash_bytes", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_Ord(__pyx_v_b); if (unlikely(__pyx_t_1 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 19, __pyx_L1_error) - __pyx_t_2 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 19, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_v_bytez, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":18 - * return b - * - * cdef _byte_to_int_py2(b): # <<<<<<<<<<<<<< - * return ord(b) - * - */ - /* function exit code */ __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("gensim.models._utils_any2vec._byte_to_int_py2", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("gensim.models._utils_any2vec.ft_hash_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":24 +/* "gensim/models/_utils_any2vec.pyx":47 * * - * cpdef ft_hash(unicode string): # <<<<<<<<<<<<<< + * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< * """Calculate hash based on `string`. - * Reproduce `hash method from Facebook fastText implementation + * */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__pyx_v_string, CYTHON_UNUSED int __pyx_skip_dispatch) { +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObject *__pyx_v_string, CYTHON_UNUSED int __pyx_skip_dispatch) { unsigned int __pyx_v_h; PyObject *__pyx_v_c = NULL; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *(*__pyx_t_3)(PyObject *); - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + void *__pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + Py_ssize_t __pyx_t_7; PyObject *__pyx_t_8 = NULL; - PyObject *__pyx_t_9 = NULL; - PyObject *__pyx_t_10 = NULL; - PyObject *__pyx_t_11 = NULL; - PyObject *__pyx_t_12 = NULL; - unsigned int __pyx_t_13; - __Pyx_RefNannySetupContext("ft_hash", 0); + long __pyx_t_9; + __Pyx_RefNannySetupContext("ft_hash_broken", 0); - /* "gensim/models/_utils_any2vec.pyx":40 + /* "gensim/models/_utils_any2vec.pyx":64 * * """ * cdef unsigned int h = 2166136261 # <<<<<<<<<<<<<< - * for c in string.encode("utf-8"): - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) + * for c in string: + * h ^= ord(c) */ __pyx_v_h = 0x811C9DC5; - /* "gensim/models/_utils_any2vec.pyx":41 + /* "gensim/models/_utils_any2vec.pyx":65 * """ * cdef unsigned int h = 2166136261 - * for c in string.encode("utf-8"): # <<<<<<<<<<<<<< - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) - * h = np.uint32(h * np.uint32(16777619)) + * for c in string: # <<<<<<<<<<<<<< + * h ^= ord(c) + * h *= 16777619 */ if (unlikely(__pyx_v_string == Py_None)) { - PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%.30s'", "encode"); - __PYX_ERR(0, 41, __pyx_L1_error) - } - __pyx_t_1 = PyUnicode_AsUTF8String(__pyx_v_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 41, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 41, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 41, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - for (;;) { - { - __pyx_t_1 = __pyx_t_3(__pyx_t_2); - if (unlikely(!__pyx_t_1)) { - PyObject* exc_type = PyErr_Occurred(); - if (exc_type) { - if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 41, __pyx_L1_error) - } - break; - } - __Pyx_GOTREF(__pyx_t_1); - } - __Pyx_XDECREF_SET(__pyx_v_c, __pyx_t_1); - __pyx_t_1 = 0; - - /* "gensim/models/_utils_any2vec.pyx":42 - * cdef unsigned int h = 2166136261 - * for c in string.encode("utf-8"): - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) # <<<<<<<<<<<<<< - * h = np.uint32(h * np.uint32(16777619)) - * return h - */ - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_np); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_uint32); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_n_s_np); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_uint32); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_n_s_np); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_9); - __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_n_s_int8); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_11, __pyx_n_s_byte_to_int); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_11); - __pyx_t_12 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_11))) { - __pyx_t_12 = PyMethod_GET_SELF(__pyx_t_11); - if (likely(__pyx_t_12)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); - __Pyx_INCREF(__pyx_t_12); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_11, function); - } - } - __pyx_t_9 = (__pyx_t_12) ? __Pyx_PyObject_Call2Args(__pyx_t_11, __pyx_t_12, __pyx_v_c) : __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_v_c); - __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; - if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_9); - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - __pyx_t_11 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_10))) { - __pyx_t_11 = PyMethod_GET_SELF(__pyx_t_10); - if (likely(__pyx_t_11)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_10); - __Pyx_INCREF(__pyx_t_11); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_10, function); - } - } - __pyx_t_7 = (__pyx_t_11) ? __Pyx_PyObject_Call2Args(__pyx_t_10, __pyx_t_11, __pyx_t_9) : __Pyx_PyObject_CallOneArg(__pyx_t_10, __pyx_t_9); - __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0; - __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __pyx_t_10 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_8))) { - __pyx_t_10 = PyMethod_GET_SELF(__pyx_t_8); - if (likely(__pyx_t_10)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); - __Pyx_INCREF(__pyx_t_10); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_8, function); - } - } - __pyx_t_6 = (__pyx_t_10) ? __Pyx_PyObject_Call2Args(__pyx_t_8, __pyx_t_10, __pyx_t_7) : __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_t_7); - __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_8 = PyNumber_Xor(__pyx_t_4, __pyx_t_6); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_5))) { - __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_5); - if (likely(__pyx_t_6)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); - __Pyx_INCREF(__pyx_t_6); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_5, function); - } - } - __pyx_t_1 = (__pyx_t_6) ? __Pyx_PyObject_Call2Args(__pyx_t_5, __pyx_t_6, __pyx_t_8) : __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_t_8); - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_13 = __Pyx_PyInt_As_unsigned_int(__pyx_t_1); if (unlikely((__pyx_t_13 == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_h = __pyx_t_13; - - /* "gensim/models/_utils_any2vec.pyx":43 - * for c in string.encode("utf-8"): - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) - * h = np.uint32(h * np.uint32(16777619)) # <<<<<<<<<<<<<< - * return h - * - */ - __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_n_s_np); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 43, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_uint32); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 43, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 43, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_np); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 43, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_uint32); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 43, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { - __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_7); - if (likely(__pyx_t_4)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); - __Pyx_INCREF(__pyx_t_4); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_7, function); - } - } - __pyx_t_6 = (__pyx_t_4) ? __Pyx_PyObject_Call2Args(__pyx_t_7, __pyx_t_4, __pyx_int_16777619) : __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_int_16777619); - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 43, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = PyNumber_Multiply(__pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 43, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_8))) { - __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_8); - if (likely(__pyx_t_6)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); - __Pyx_INCREF(__pyx_t_6); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_8, function); - } - } - __pyx_t_1 = (__pyx_t_6) ? __Pyx_PyObject_Call2Args(__pyx_t_8, __pyx_t_6, __pyx_t_7) : __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_t_7); - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 43, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_13 = __Pyx_PyInt_As_unsigned_int(__pyx_t_1); if (unlikely((__pyx_t_13 == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 43, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_h = __pyx_t_13; - - /* "gensim/models/_utils_any2vec.pyx":41 - * """ - * cdef unsigned int h = 2166136261 - * for c in string.encode("utf-8"): # <<<<<<<<<<<<<< - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) - * h = np.uint32(h * np.uint32(16777619)) - */ - } - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "gensim/models/_utils_any2vec.pyx":44 - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) - * h = np.uint32(h * np.uint32(16777619)) - * return h # <<<<<<<<<<<<<< - * - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 44, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; - goto __pyx_L0; - - /* "gensim/models/_utils_any2vec.pyx":24 - * - * - * cpdef ft_hash(unicode string): # <<<<<<<<<<<<<< - * """Calculate hash based on `string`. - * Reproduce `hash method from Facebook fastText implementation - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_XDECREF(__pyx_t_9); - __Pyx_XDECREF(__pyx_t_10); - __Pyx_XDECREF(__pyx_t_11); - __Pyx_XDECREF(__pyx_t_12); - __Pyx_AddTraceback("gensim.models._utils_any2vec.ft_hash", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_c); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* Python wrapper */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ -static char __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash[] = "ft_hash(unicode string)\nCalculate hash based on `string`.\n Reproduce `hash method from Facebook fastText implementation\n `_.\n\n Parameters\n ----------\n string : unicode\n The string whose hash needs to be calculated.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash(PyObject *__pyx_self, PyObject *__pyx_v_string) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("ft_hash (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 24, __pyx_L1_error) - __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash(__pyx_self, ((PyObject*)__pyx_v_string)); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("ft_hash", 0); - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 24, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("gensim.models._utils_any2vec.ft_hash", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "gensim/models/_utils_any2vec.pyx":47 - * - * - * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< - * """Calculate hash based on `string`. - * - */ - -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObject *__pyx_v_string, CYTHON_UNUSED int __pyx_skip_dispatch) { - unsigned int __pyx_v_h; - PyObject *__pyx_v_c = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - Py_ssize_t __pyx_t_2; - Py_ssize_t __pyx_t_3; - void *__pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - Py_ssize_t __pyx_t_7; - PyObject *__pyx_t_8 = NULL; - long __pyx_t_9; - __Pyx_RefNannySetupContext("ft_hash_broken", 0); - - /* "gensim/models/_utils_any2vec.pyx":64 - * - * """ - * cdef unsigned int h = 2166136261 # <<<<<<<<<<<<<< - * for c in string: - * h ^= ord(c) - */ - __pyx_v_h = 0x811C9DC5; - - /* "gensim/models/_utils_any2vec.pyx":65 - * """ - * cdef unsigned int h = 2166136261 - * for c in string: # <<<<<<<<<<<<<< - * h ^= ord(c) - * h *= 16777619 - */ - if (unlikely(__pyx_v_string == Py_None)) { - PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); - __PYX_ERR(0, 65, __pyx_L1_error) + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(0, 65, __pyx_L1_error) } __Pyx_INCREF(__pyx_v_string); __pyx_t_1 = __pyx_v_string; @@ -2610,6 +2240,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) # <<<<<<<<<<<<<< * return ngrams + * */ __Pyx_INCREF(__pyx_v_i); __pyx_t_1 = __pyx_v_i; @@ -2660,6 +2291,8 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams # <<<<<<<<<<<<<< + * + * # */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_ngrams); @@ -2787,108 +2420,531 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHO return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 - * # experimental exception made for __getbuffer__ and __releasebuffer__ - * # -- the details of this may change. - * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< - * # This implementation of getbuffer is geared towards Cython - * # requirements, and does not yet fulfill the PEP. +/* "gensim/models/_utils_any2vec.pyx":104 + * + * + * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< + * """Computes ngrams for a word. + * */ -/* Python wrapper */ -static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ -static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { - int __pyx_r; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n, CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_v_utf8_word = 0; + unsigned char const *__pyx_v_bytez; + size_t __pyx_v_num_bytes; + size_t __pyx_v_j; + size_t __pyx_v_i; + size_t __pyx_v_n; + PyObject *__pyx_v_ngrams = NULL; + PyObject *__pyx_v_ngram = NULL; + PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); - __pyx_r = __pyx_pf_5numpy_7ndarray___getbuffer__(((PyArrayObject *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + unsigned char const *__pyx_t_4; + Py_ssize_t __pyx_t_5; + size_t __pyx_t_6; + size_t __pyx_t_7; + size_t __pyx_t_8; + int __pyx_t_9; + size_t __pyx_t_10; + size_t __pyx_t_11; + int __pyx_t_12; + int __pyx_t_13; + int __pyx_t_14; + __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} + /* "gensim/models/_utils_any2vec.pyx":128 + * + * """ + * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") # <<<<<<<<<<<<<< + * cdef const unsigned char *bytez = utf8_word + * cdef size_t num_bytes = len(utf8_word) + */ + __pyx_t_2 = __Pyx_PyString_FormatSafe(__pyx_kp_s_s, __pyx_v_word); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 128, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 128, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_2)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + __pyx_t_1 = (__pyx_t_2) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_2, __pyx_kp_s_utf_8) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_kp_s_utf_8); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 128, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 128, __pyx_L1_error) + __pyx_v_utf8_word = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; -static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { - int __pyx_v_i; - int __pyx_v_ndim; - int __pyx_v_endian_detector; - int __pyx_v_little_endian; - int __pyx_v_t; - char *__pyx_v_f; - PyArray_Descr *__pyx_v_descr = 0; - int __pyx_v_offset; - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - int __pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - PyArray_Descr *__pyx_t_7; - PyObject *__pyx_t_8 = NULL; - char *__pyx_t_9; - if (__pyx_v_info == NULL) { - PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); - return -1; + /* "gensim/models/_utils_any2vec.pyx":129 + * """ + * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") + * cdef const unsigned char *bytez = utf8_word # <<<<<<<<<<<<<< + * cdef size_t num_bytes = len(utf8_word) + * cdef size_t j, i, n + */ + if (unlikely(__pyx_v_utf8_word == Py_None)) { + PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); + __PYX_ERR(0, 129, __pyx_L1_error) } - __Pyx_RefNannySetupContext("__getbuffer__", 0); - __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); - __Pyx_GIVEREF(__pyx_v_info->obj); + __pyx_t_4 = __Pyx_PyBytes_AsUString(__pyx_v_utf8_word); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) __PYX_ERR(0, 129, __pyx_L1_error) + __pyx_v_bytez = __pyx_t_4; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 - * - * cdef int i, ndim - * cdef int endian_detector = 1 # <<<<<<<<<<<<<< - * cdef bint little_endian = ((&endian_detector)[0] != 0) + /* "gensim/models/_utils_any2vec.pyx":130 + * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") + * cdef const unsigned char *bytez = utf8_word + * cdef size_t num_bytes = len(utf8_word) # <<<<<<<<<<<<<< + * cdef size_t j, i, n * */ - __pyx_v_endian_detector = 1; + if (unlikely(__pyx_v_utf8_word == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 130, __pyx_L1_error) + } + __pyx_t_5 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 130, __pyx_L1_error) + __pyx_v_num_bytes = __pyx_t_5; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 - * cdef int i, ndim - * cdef int endian_detector = 1 - * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< + /* "gensim/models/_utils_any2vec.pyx":133 + * cdef size_t j, i, n * - * ndim = PyArray_NDIM(self) + * ngrams = [] # <<<<<<<<<<<<<< + * for i in range(num_bytes): + * if bytez[i] & _MB_MASK == _MB_START: */ - __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 133, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_ngrams = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 - * cdef bint little_endian = ((&endian_detector)[0] != 0) - * - * ndim = PyArray_NDIM(self) # <<<<<<<<<<<<<< + /* "gensim/models/_utils_any2vec.pyx":134 * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) + * ngrams = [] + * for i in range(num_bytes): # <<<<<<<<<<<<<< + * if bytez[i] & _MB_MASK == _MB_START: + * continue */ - __pyx_v_ndim = PyArray_NDIM(__pyx_v_self); + __pyx_t_6 = __pyx_v_num_bytes; + __pyx_t_7 = __pyx_t_6; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { + __pyx_v_i = __pyx_t_8; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 - * ndim = PyArray_NDIM(self) + /* "gensim/models/_utils_any2vec.pyx":135 + * ngrams = [] + * for i in range(num_bytes): + * if bytez[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< + * continue * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): - * raise ValueError(u"ndarray is not C contiguous") */ - __pyx_t_2 = (((__pyx_v_flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) != 0); - if (__pyx_t_2) { - } else { - __pyx_t_1 = __pyx_t_2; - goto __pyx_L4_bool_binop_done; - } + __pyx_t_9 = ((((__pyx_v_bytez[__pyx_v_i]) & __pyx_v_6gensim_6models_14_utils_any2vec__MB_MASK) == __pyx_v_6gensim_6models_14_utils_any2vec__MB_START) != 0); + if (__pyx_t_9) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 + /* "gensim/models/_utils_any2vec.pyx":136 + * for i in range(num_bytes): + * if bytez[i] & _MB_MASK == _MB_START: + * continue # <<<<<<<<<<<<<< * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): # <<<<<<<<<<<<<< - * raise ValueError(u"ndarray is not C contiguous") + * j, n = i, 1 + */ + goto __pyx_L3_continue; + + /* "gensim/models/_utils_any2vec.pyx":135 + * ngrams = [] + * for i in range(num_bytes): + * if bytez[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< + * continue + * + */ + } + + /* "gensim/models/_utils_any2vec.pyx":138 + * continue + * + * j, n = i, 1 # <<<<<<<<<<<<<< + * while j < num_bytes and n <= max_n: + * j += 1 + */ + __pyx_t_10 = __pyx_v_i; + __pyx_t_11 = 1; + __pyx_v_j = __pyx_t_10; + __pyx_v_n = __pyx_t_11; + + /* "gensim/models/_utils_any2vec.pyx":139 + * + * j, n = i, 1 + * while j < num_bytes and n <= max_n: # <<<<<<<<<<<<<< + * j += 1 + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: + */ + while (1) { + __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); + if (__pyx_t_12) { + } else { + __pyx_t_9 = __pyx_t_12; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_12 = ((__pyx_v_n <= __pyx_v_max_n) != 0); + __pyx_t_9 = __pyx_t_12; + __pyx_L8_bool_binop_done:; + if (!__pyx_t_9) break; + + /* "gensim/models/_utils_any2vec.pyx":140 + * j, n = i, 1 + * while j < num_bytes and n <= max_n: + * j += 1 # <<<<<<<<<<<<<< + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: + * j += 1 + */ + __pyx_v_j = (__pyx_v_j + 1); + + /* "gensim/models/_utils_any2vec.pyx":141 + * while j < num_bytes and n <= max_n: + * j += 1 + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: # <<<<<<<<<<<<<< + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + */ + while (1) { + __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); + if (__pyx_t_12) { + } else { + __pyx_t_9 = __pyx_t_12; + goto __pyx_L12_bool_binop_done; + } + __pyx_t_12 = ((((__pyx_v_bytez[__pyx_v_j]) & __pyx_v_6gensim_6models_14_utils_any2vec__MB_MASK) == __pyx_v_6gensim_6models_14_utils_any2vec__MB_START) != 0); + __pyx_t_9 = __pyx_t_12; + __pyx_L12_bool_binop_done:; + if (!__pyx_t_9) break; + + /* "gensim/models/_utils_any2vec.pyx":142 + * j += 1 + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: + * j += 1 # <<<<<<<<<<<<<< + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + * ngram = bytes(bytez[i:j]) + */ + __pyx_v_j = (__pyx_v_j + 1); + } + + /* "gensim/models/_utils_any2vec.pyx":143 + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< + * ngram = bytes(bytez[i:j]) + * ngrams.append(ngram) + */ + __pyx_t_12 = ((__pyx_v_n >= __pyx_v_min_n) != 0); + if (__pyx_t_12) { + } else { + __pyx_t_9 = __pyx_t_12; + goto __pyx_L15_bool_binop_done; + } + __pyx_t_13 = ((__pyx_v_n == 1) != 0); + if (__pyx_t_13) { + } else { + __pyx_t_12 = __pyx_t_13; + goto __pyx_L17_bool_binop_done; + } + __pyx_t_13 = ((__pyx_v_i == 0) != 0); + if (!__pyx_t_13) { + } else { + __pyx_t_12 = __pyx_t_13; + goto __pyx_L17_bool_binop_done; + } + __pyx_t_13 = ((__pyx_v_j == __pyx_v_num_bytes) != 0); + __pyx_t_12 = __pyx_t_13; + __pyx_L17_bool_binop_done:; + __pyx_t_13 = ((!__pyx_t_12) != 0); + __pyx_t_9 = __pyx_t_13; + __pyx_L15_bool_binop_done:; + if (__pyx_t_9) { + + /* "gensim/models/_utils_any2vec.pyx":144 + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + * ngram = bytes(bytez[i:j]) # <<<<<<<<<<<<<< + * ngrams.append(ngram) + * n += 1 + */ + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(((const char*)__pyx_v_bytez) + __pyx_v_i, __pyx_v_j - __pyx_v_i); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 144, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 144, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF_SET(__pyx_v_ngram, ((PyObject*)__pyx_t_3)); + __pyx_t_3 = 0; + + /* "gensim/models/_utils_any2vec.pyx":145 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + * ngram = bytes(bytez[i:j]) + * ngrams.append(ngram) # <<<<<<<<<<<<<< + * n += 1 + * return ngrams + */ + __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_v_ngram); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 145, __pyx_L1_error) + + /* "gensim/models/_utils_any2vec.pyx":143 + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< + * ngram = bytes(bytez[i:j]) + * ngrams.append(ngram) + */ + } + + /* "gensim/models/_utils_any2vec.pyx":146 + * ngram = bytes(bytez[i:j]) + * ngrams.append(ngram) + * n += 1 # <<<<<<<<<<<<<< + * return ngrams + */ + __pyx_v_n = (__pyx_v_n + 1); + } + __pyx_L3_continue:; + } + + /* "gensim/models/_utils_any2vec.pyx":147 + * ngrams.append(ngram) + * n += 1 + * return ngrams # <<<<<<<<<<<<<< + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_ngrams); + __pyx_r = __pyx_v_ngrams; + goto __pyx_L0; + + /* "gensim/models/_utils_any2vec.pyx":104 + * + * + * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< + * """Computes ngrams for a word. + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_utf8_word); + __Pyx_XDECREF(__pyx_v_ngrams); + __Pyx_XDECREF(__pyx_v_ngram); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes[] = "compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n)\nComputes ngrams for a word.\n\n Ported from the original FB implementation.\n\n Parameters\n ----------\n word : str\n A unicode string.\n min_n : unsigned int\n The minimum ngram length.\n max_n : unsigned int\n The maximum ngram length.\n\n Returns:\n --------\n list of str\n A list of ngrams, where each ngram is a list of **bytes**.\n\n See Also\n --------\n `Original implementation `__\n\n "; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_word = 0; + unsigned int __pyx_v_min_n; + unsigned int __pyx_v_max_n; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("compute_ngrams_bytes (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_word,&__pyx_n_s_min_n,&__pyx_n_s_max_n,0}; + PyObject* values[3] = {0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_word)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 1); __PYX_ERR(0, 104, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 2); __PYX_ERR(0, 104, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes") < 0)) __PYX_ERR(0, 104, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + } + __pyx_v_word = values[0]; + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 104, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 104, __pyx_L3_error) + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 104, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes(__pyx_self, __pyx_v_word, __pyx_v_min_n, __pyx_v_max_n); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 104, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 + * # experimental exception made for __getbuffer__ and __releasebuffer__ + * # -- the details of this may change. + * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< + * # This implementation of getbuffer is geared towards Cython + * # requirements, and does not yet fulfill the PEP. + */ + +/* Python wrapper */ +static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); + __pyx_r = __pyx_pf_5numpy_7ndarray___getbuffer__(((PyArrayObject *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_v_i; + int __pyx_v_ndim; + int __pyx_v_endian_detector; + int __pyx_v_little_endian; + int __pyx_v_t; + char *__pyx_v_f; + PyArray_Descr *__pyx_v_descr = 0; + int __pyx_v_offset; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + PyArray_Descr *__pyx_t_7; + PyObject *__pyx_t_8 = NULL; + char *__pyx_t_9; + if (__pyx_v_info == NULL) { + PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); + return -1; + } + __Pyx_RefNannySetupContext("__getbuffer__", 0); + __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(__pyx_v_info->obj); + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 + * + * cdef int i, ndim + * cdef int endian_detector = 1 # <<<<<<<<<<<<<< + * cdef bint little_endian = ((&endian_detector)[0] != 0) + * + */ + __pyx_v_endian_detector = 1; + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 + * cdef int i, ndim + * cdef int endian_detector = 1 + * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< + * + * ndim = PyArray_NDIM(self) + */ + __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 + * cdef bint little_endian = ((&endian_detector)[0] != 0) + * + * ndim = PyArray_NDIM(self) # <<<<<<<<<<<<<< + * + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) + */ + __pyx_v_ndim = PyArray_NDIM(__pyx_v_self); + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + * ndim = PyArray_NDIM(self) + * + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): + * raise ValueError(u"ndarray is not C contiguous") + */ + __pyx_t_2 = (((__pyx_v_flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) != 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 + * + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): # <<<<<<<<<<<<<< + * raise ValueError(u"ndarray is not C contiguous") * */ __pyx_t_2 = ((!(PyArray_CHKFLAGS(__pyx_v_self, NPY_ARRAY_C_CONTIGUOUS) != 0)) != 0); __pyx_t_1 = __pyx_t_2; __pyx_L4_bool_binop_done:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 * ndim = PyArray_NDIM(self) * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2897,7 +2953,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< @@ -2910,7 +2966,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 272, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 * ndim = PyArray_NDIM(self) * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2919,7 +2975,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2933,7 +2989,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P goto __pyx_L7_bool_binop_done; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":275 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":275 * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): # <<<<<<<<<<<<<< @@ -2944,7 +3000,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = __pyx_t_2; __pyx_L7_bool_binop_done:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2953,7 +3009,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< @@ -2966,7 +3022,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 276, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2975,7 +3031,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":278 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":278 * raise ValueError(u"ndarray is not Fortran contiguous") * * info.buf = PyArray_DATA(self) # <<<<<<<<<<<<<< @@ -2984,7 +3040,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->buf = PyArray_DATA(__pyx_v_self); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":279 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":279 * * info.buf = PyArray_DATA(self) * info.ndim = ndim # <<<<<<<<<<<<<< @@ -2993,7 +3049,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->ndim = __pyx_v_ndim; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 * info.buf = PyArray_DATA(self) * info.ndim = ndim * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3003,7 +3059,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":283 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":283 * # Allocate new buffer for strides and shape info. * # This is allocated as one block, strides first. * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) # <<<<<<<<<<<<<< @@ -3012,7 +3068,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->strides = ((Py_ssize_t *)PyObject_Malloc((((sizeof(Py_ssize_t)) * 2) * ((size_t)__pyx_v_ndim)))); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":284 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":284 * # This is allocated as one block, strides first. * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) * info.shape = info.strides + ndim # <<<<<<<<<<<<<< @@ -3021,7 +3077,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->shape = (__pyx_v_info->strides + __pyx_v_ndim); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":285 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":285 * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) * info.shape = info.strides + ndim * for i in range(ndim): # <<<<<<<<<<<<<< @@ -3033,7 +3089,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { __pyx_v_i = __pyx_t_6; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":286 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":286 * info.shape = info.strides + ndim * for i in range(ndim): * info.strides[i] = PyArray_STRIDES(self)[i] # <<<<<<<<<<<<<< @@ -3042,7 +3098,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ (__pyx_v_info->strides[__pyx_v_i]) = (PyArray_STRIDES(__pyx_v_self)[__pyx_v_i]); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":287 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":287 * for i in range(ndim): * info.strides[i] = PyArray_STRIDES(self)[i] * info.shape[i] = PyArray_DIMS(self)[i] # <<<<<<<<<<<<<< @@ -3052,7 +3108,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P (__pyx_v_info->shape[__pyx_v_i]) = (PyArray_DIMS(__pyx_v_self)[__pyx_v_i]); } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 * info.buf = PyArray_DATA(self) * info.ndim = ndim * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3062,7 +3118,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P goto __pyx_L9; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":289 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":289 * info.shape[i] = PyArray_DIMS(self)[i] * else: * info.strides = PyArray_STRIDES(self) # <<<<<<<<<<<<<< @@ -3072,7 +3128,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P /*else*/ { __pyx_v_info->strides = ((Py_ssize_t *)PyArray_STRIDES(__pyx_v_self)); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":290 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":290 * else: * info.strides = PyArray_STRIDES(self) * info.shape = PyArray_DIMS(self) # <<<<<<<<<<<<<< @@ -3083,7 +3139,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P } __pyx_L9:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":291 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":291 * info.strides = PyArray_STRIDES(self) * info.shape = PyArray_DIMS(self) * info.suboffsets = NULL # <<<<<<<<<<<<<< @@ -3092,7 +3148,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->suboffsets = NULL; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":292 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":292 * info.shape = PyArray_DIMS(self) * info.suboffsets = NULL * info.itemsize = PyArray_ITEMSIZE(self) # <<<<<<<<<<<<<< @@ -3101,7 +3157,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->itemsize = PyArray_ITEMSIZE(__pyx_v_self); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":293 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":293 * info.suboffsets = NULL * info.itemsize = PyArray_ITEMSIZE(self) * info.readonly = not PyArray_ISWRITEABLE(self) # <<<<<<<<<<<<<< @@ -3110,7 +3166,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->readonly = (!(PyArray_ISWRITEABLE(__pyx_v_self) != 0)); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":296 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":296 * * cdef int t * cdef char* f = NULL # <<<<<<<<<<<<<< @@ -3119,7 +3175,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_f = NULL; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":297 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":297 * cdef int t * cdef char* f = NULL * cdef dtype descr = PyArray_DESCR(self) # <<<<<<<<<<<<<< @@ -3132,7 +3188,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_v_descr = ((PyArray_Descr *)__pyx_t_3); __pyx_t_3 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":300 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":300 * cdef int offset * * info.obj = self # <<<<<<<<<<<<<< @@ -3145,7 +3201,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = ((PyObject *)__pyx_v_self); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 * info.obj = self * * if not PyDataType_HASFIELDS(descr): # <<<<<<<<<<<<<< @@ -3155,7 +3211,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = ((!(PyDataType_HASFIELDS(__pyx_v_descr) != 0)) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":303 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":303 * * if not PyDataType_HASFIELDS(descr): * t = descr.type_num # <<<<<<<<<<<<<< @@ -3165,7 +3221,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_4 = __pyx_v_descr->type_num; __pyx_v_t = __pyx_t_4; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3185,7 +3241,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P } __pyx_L15_next_or:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":305 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":305 * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< @@ -3202,7 +3258,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = __pyx_t_2; __pyx_L14_bool_binop_done:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3211,7 +3267,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -3224,7 +3280,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 306, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3233,7 +3289,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":307 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":307 * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") * if t == NPY_BYTE: f = "b" # <<<<<<<<<<<<<< @@ -3246,7 +3302,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_UBYTE: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":308 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":308 * raise ValueError(u"Non-native byte order not supported") * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" # <<<<<<<<<<<<<< @@ -3257,7 +3313,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_SHORT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":309 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":309 * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" * elif t == NPY_SHORT: f = "h" # <<<<<<<<<<<<<< @@ -3268,7 +3324,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_USHORT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":310 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":310 * elif t == NPY_UBYTE: f = "B" * elif t == NPY_SHORT: f = "h" * elif t == NPY_USHORT: f = "H" # <<<<<<<<<<<<<< @@ -3279,7 +3335,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_INT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":311 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":311 * elif t == NPY_SHORT: f = "h" * elif t == NPY_USHORT: f = "H" * elif t == NPY_INT: f = "i" # <<<<<<<<<<<<<< @@ -3290,7 +3346,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_UINT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":312 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":312 * elif t == NPY_USHORT: f = "H" * elif t == NPY_INT: f = "i" * elif t == NPY_UINT: f = "I" # <<<<<<<<<<<<<< @@ -3301,7 +3357,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONG: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":313 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":313 * elif t == NPY_INT: f = "i" * elif t == NPY_UINT: f = "I" * elif t == NPY_LONG: f = "l" # <<<<<<<<<<<<<< @@ -3312,7 +3368,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_ULONG: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":314 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":314 * elif t == NPY_UINT: f = "I" * elif t == NPY_LONG: f = "l" * elif t == NPY_ULONG: f = "L" # <<<<<<<<<<<<<< @@ -3323,7 +3379,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONGLONG: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":315 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":315 * elif t == NPY_LONG: f = "l" * elif t == NPY_ULONG: f = "L" * elif t == NPY_LONGLONG: f = "q" # <<<<<<<<<<<<<< @@ -3334,7 +3390,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_ULONGLONG: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":316 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":316 * elif t == NPY_ULONG: f = "L" * elif t == NPY_LONGLONG: f = "q" * elif t == NPY_ULONGLONG: f = "Q" # <<<<<<<<<<<<<< @@ -3345,7 +3401,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_FLOAT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":317 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":317 * elif t == NPY_LONGLONG: f = "q" * elif t == NPY_ULONGLONG: f = "Q" * elif t == NPY_FLOAT: f = "f" # <<<<<<<<<<<<<< @@ -3356,7 +3412,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_DOUBLE: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":318 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":318 * elif t == NPY_ULONGLONG: f = "Q" * elif t == NPY_FLOAT: f = "f" * elif t == NPY_DOUBLE: f = "d" # <<<<<<<<<<<<<< @@ -3367,7 +3423,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONGDOUBLE: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":319 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":319 * elif t == NPY_FLOAT: f = "f" * elif t == NPY_DOUBLE: f = "d" * elif t == NPY_LONGDOUBLE: f = "g" # <<<<<<<<<<<<<< @@ -3378,7 +3434,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CFLOAT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":320 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":320 * elif t == NPY_DOUBLE: f = "d" * elif t == NPY_LONGDOUBLE: f = "g" * elif t == NPY_CFLOAT: f = "Zf" # <<<<<<<<<<<<<< @@ -3389,7 +3445,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CDOUBLE: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":321 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":321 * elif t == NPY_LONGDOUBLE: f = "g" * elif t == NPY_CFLOAT: f = "Zf" * elif t == NPY_CDOUBLE: f = "Zd" # <<<<<<<<<<<<<< @@ -3400,7 +3456,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CLONGDOUBLE: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":322 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":322 * elif t == NPY_CFLOAT: f = "Zf" * elif t == NPY_CDOUBLE: f = "Zd" * elif t == NPY_CLONGDOUBLE: f = "Zg" # <<<<<<<<<<<<<< @@ -3411,7 +3467,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_OBJECT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":323 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":323 * elif t == NPY_CDOUBLE: f = "Zd" * elif t == NPY_CLONGDOUBLE: f = "Zg" * elif t == NPY_OBJECT: f = "O" # <<<<<<<<<<<<<< @@ -3422,7 +3478,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; default: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":325 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":325 * elif t == NPY_OBJECT: f = "O" * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< @@ -3443,7 +3499,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":326 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":326 * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * info.format = f # <<<<<<<<<<<<<< @@ -3452,7 +3508,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->format = __pyx_v_f; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":327 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":327 * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * info.format = f * return # <<<<<<<<<<<<<< @@ -3462,7 +3518,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_r = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 * info.obj = self * * if not PyDataType_HASFIELDS(descr): # <<<<<<<<<<<<<< @@ -3471,7 +3527,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":329 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":329 * return * else: * info.format = PyObject_Malloc(_buffer_format_string_len) # <<<<<<<<<<<<<< @@ -3481,7 +3537,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P /*else*/ { __pyx_v_info->format = ((char *)PyObject_Malloc(0xFF)); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":330 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":330 * else: * info.format = PyObject_Malloc(_buffer_format_string_len) * info.format[0] = c'^' # Native data types, manual alignment # <<<<<<<<<<<<<< @@ -3490,7 +3546,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ (__pyx_v_info->format[0]) = '^'; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":331 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":331 * info.format = PyObject_Malloc(_buffer_format_string_len) * info.format[0] = c'^' # Native data types, manual alignment * offset = 0 # <<<<<<<<<<<<<< @@ -3499,7 +3555,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_offset = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":332 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":332 * info.format[0] = c'^' # Native data types, manual alignment * offset = 0 * f = _util_dtypestring(descr, info.format + 1, # <<<<<<<<<<<<<< @@ -3509,7 +3565,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_9 = __pyx_f_5numpy__util_dtypestring(__pyx_v_descr, (__pyx_v_info->format + 1), (__pyx_v_info->format + 0xFF), (&__pyx_v_offset)); if (unlikely(__pyx_t_9 == ((char *)NULL))) __PYX_ERR(1, 332, __pyx_L1_error) __pyx_v_f = __pyx_t_9; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":335 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":335 * info.format + _buffer_format_string_len, * &offset) * f[0] = c'\0' # Terminate format string # <<<<<<<<<<<<<< @@ -3519,7 +3575,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P (__pyx_v_f[0]) = '\x00'; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 * # experimental exception made for __getbuffer__ and __releasebuffer__ * # -- the details of this may change. * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< @@ -3551,7 +3607,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 * f[0] = c'\0' # Terminate format string * * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< @@ -3575,7 +3631,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s int __pyx_t_1; __Pyx_RefNannySetupContext("__releasebuffer__", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 * * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< @@ -3585,7 +3641,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __pyx_t_1 = (PyArray_HASFIELDS(__pyx_v_self) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":339 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":339 * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) # <<<<<<<<<<<<<< @@ -3594,7 +3650,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ PyObject_Free(__pyx_v_info->format); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 * * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< @@ -3603,7 +3659,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3613,7 +3669,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":341 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":341 * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): * PyObject_Free(info.strides) # <<<<<<<<<<<<<< @@ -3622,7 +3678,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ PyObject_Free(__pyx_v_info->strides); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3631,7 +3687,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 * f[0] = c'\0' # Terminate format string * * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< @@ -3643,7 +3699,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __Pyx_RefNannyFinishContext(); } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 * ctypedef npy_cdouble complex_t * * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< @@ -3657,7 +3713,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew1", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":822 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":822 * * cdef inline object PyArray_MultiIterNew1(a): * return PyArray_MultiIterNew(1, a) # <<<<<<<<<<<<<< @@ -3671,7 +3727,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 * ctypedef npy_cdouble complex_t * * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< @@ -3690,7 +3746,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 * return PyArray_MultiIterNew(1, a) * * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< @@ -3704,7 +3760,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew2", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":825 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":825 * * cdef inline object PyArray_MultiIterNew2(a, b): * return PyArray_MultiIterNew(2, a, b) # <<<<<<<<<<<<<< @@ -3718,7 +3774,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 * return PyArray_MultiIterNew(1, a) * * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< @@ -3737,7 +3793,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 * return PyArray_MultiIterNew(2, a, b) * * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< @@ -3751,7 +3807,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew3", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":828 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":828 * * cdef inline object PyArray_MultiIterNew3(a, b, c): * return PyArray_MultiIterNew(3, a, b, c) # <<<<<<<<<<<<<< @@ -3765,7 +3821,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 * return PyArray_MultiIterNew(2, a, b) * * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< @@ -3784,7 +3840,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 * return PyArray_MultiIterNew(3, a, b, c) * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< @@ -3798,7 +3854,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew4", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":831 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":831 * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): * return PyArray_MultiIterNew(4, a, b, c, d) # <<<<<<<<<<<<<< @@ -3812,7 +3868,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 * return PyArray_MultiIterNew(3, a, b, c) * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< @@ -3831,7 +3887,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 * return PyArray_MultiIterNew(4, a, b, c, d) * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< @@ -3845,7 +3901,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew5", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":834 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":834 * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): * return PyArray_MultiIterNew(5, a, b, c, d, e) # <<<<<<<<<<<<<< @@ -3859,7 +3915,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 * return PyArray_MultiIterNew(4, a, b, c, d) * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< @@ -3878,7 +3934,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 * return PyArray_MultiIterNew(5, a, b, c, d, e) * * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< @@ -3892,7 +3948,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ int __pyx_t_1; __Pyx_RefNannySetupContext("PyDataType_SHAPE", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 * * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< @@ -3902,7 +3958,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ __pyx_t_1 = (PyDataType_HASSUBARRAY(__pyx_v_d) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":838 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":838 * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): * return d.subarray.shape # <<<<<<<<<<<<<< @@ -3914,7 +3970,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ __pyx_r = ((PyObject*)__pyx_v_d->subarray->shape); goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 * * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< @@ -3923,7 +3979,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":840 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":840 * return d.subarray.shape * else: * return () # <<<<<<<<<<<<<< @@ -3937,7 +3993,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ goto __pyx_L0; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 * return PyArray_MultiIterNew(5, a, b, c, d, e) * * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< @@ -3952,7 +4008,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 * return () * * cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: # <<<<<<<<<<<<<< @@ -3981,7 +4037,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx char *__pyx_t_9; __Pyx_RefNannySetupContext("_util_dtypestring", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":847 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":847 * * cdef dtype child * cdef int endian_detector = 1 # <<<<<<<<<<<<<< @@ -3990,7 +4046,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_endian_detector = 1; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":848 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":848 * cdef dtype child * cdef int endian_detector = 1 * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< @@ -3999,7 +4055,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 * cdef tuple fields * * for childname in descr.names: # <<<<<<<<<<<<<< @@ -4022,7 +4078,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_childname, __pyx_t_3); __pyx_t_3 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":852 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":852 * * for childname in descr.names: * fields = descr.fields[childname] # <<<<<<<<<<<<<< @@ -4039,7 +4095,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_fields, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":853 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":853 * for childname in descr.names: * fields = descr.fields[childname] * child, new_offset = fields # <<<<<<<<<<<<<< @@ -4074,7 +4130,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_new_offset, __pyx_t_4); __pyx_t_4 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 * child, new_offset = fields * * if (end - f) - (new_offset - offset[0]) < 15: # <<<<<<<<<<<<<< @@ -4091,7 +4147,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = ((((__pyx_v_end - __pyx_v_f) - ((int)__pyx_t_5)) < 15) != 0); if (unlikely(__pyx_t_6)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 * * if (end - f) - (new_offset - offset[0]) < 15: * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") # <<<<<<<<<<<<<< @@ -4104,7 +4160,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 856, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 * child, new_offset = fields * * if (end - f) - (new_offset - offset[0]) < 15: # <<<<<<<<<<<<<< @@ -4113,7 +4169,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4133,7 +4189,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L8_next_or:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":859 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":859 * * if ((child.byteorder == c'>' and little_endian) or * (child.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< @@ -4150,7 +4206,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = __pyx_t_7; __pyx_L7_bool_binop_done:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4159,7 +4215,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ if (unlikely(__pyx_t_6)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":860 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":860 * if ((child.byteorder == c'>' and little_endian) or * (child.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -4172,7 +4228,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 860, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4181,7 +4237,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":870 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":870 * * # Output padding bytes * while offset[0] < new_offset: # <<<<<<<<<<<<<< @@ -4197,7 +4253,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (!__pyx_t_6) break; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":871 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":871 * # Output padding bytes * while offset[0] < new_offset: * f[0] = 120 # "x"; pad byte # <<<<<<<<<<<<<< @@ -4206,7 +4262,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ (__pyx_v_f[0]) = 0x78; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":872 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":872 * while offset[0] < new_offset: * f[0] = 120 # "x"; pad byte * f += 1 # <<<<<<<<<<<<<< @@ -4215,7 +4271,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_f = (__pyx_v_f + 1); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":873 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":873 * f[0] = 120 # "x"; pad byte * f += 1 * offset[0] += 1 # <<<<<<<<<<<<<< @@ -4226,7 +4282,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx (__pyx_v_offset[__pyx_t_8]) = ((__pyx_v_offset[__pyx_t_8]) + 1); } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":875 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":875 * offset[0] += 1 * * offset[0] += child.itemsize # <<<<<<<<<<<<<< @@ -4236,7 +4292,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_8 = 0; (__pyx_v_offset[__pyx_t_8]) = ((__pyx_v_offset[__pyx_t_8]) + __pyx_v_child->elsize); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 * offset[0] += child.itemsize * * if not PyDataType_HASFIELDS(child): # <<<<<<<<<<<<<< @@ -4246,7 +4302,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = ((!(PyDataType_HASFIELDS(__pyx_v_child) != 0)) != 0); if (__pyx_t_6) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":878 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":878 * * if not PyDataType_HASFIELDS(child): * t = child.type_num # <<<<<<<<<<<<<< @@ -4258,7 +4314,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_t, __pyx_t_4); __pyx_t_4 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 * if not PyDataType_HASFIELDS(child): * t = child.type_num * if end - f < 5: # <<<<<<<<<<<<<< @@ -4268,7 +4324,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = (((__pyx_v_end - __pyx_v_f) < 5) != 0); if (unlikely(__pyx_t_6)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 * t = child.type_num * if end - f < 5: * raise RuntimeError(u"Format string allocated too short.") # <<<<<<<<<<<<<< @@ -4281,7 +4337,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __PYX_ERR(1, 880, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 * if not PyDataType_HASFIELDS(child): * t = child.type_num * if end - f < 5: # <<<<<<<<<<<<<< @@ -4290,7 +4346,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":883 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":883 * * # Until ticket #99 is fixed, use integers to avoid warnings * if t == NPY_BYTE: f[0] = 98 #"b" # <<<<<<<<<<<<<< @@ -4308,7 +4364,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":884 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":884 * # Until ticket #99 is fixed, use integers to avoid warnings * if t == NPY_BYTE: f[0] = 98 #"b" * elif t == NPY_UBYTE: f[0] = 66 #"B" # <<<<<<<<<<<<<< @@ -4326,7 +4382,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":885 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":885 * if t == NPY_BYTE: f[0] = 98 #"b" * elif t == NPY_UBYTE: f[0] = 66 #"B" * elif t == NPY_SHORT: f[0] = 104 #"h" # <<<<<<<<<<<<<< @@ -4344,7 +4400,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":886 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":886 * elif t == NPY_UBYTE: f[0] = 66 #"B" * elif t == NPY_SHORT: f[0] = 104 #"h" * elif t == NPY_USHORT: f[0] = 72 #"H" # <<<<<<<<<<<<<< @@ -4362,7 +4418,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":887 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":887 * elif t == NPY_SHORT: f[0] = 104 #"h" * elif t == NPY_USHORT: f[0] = 72 #"H" * elif t == NPY_INT: f[0] = 105 #"i" # <<<<<<<<<<<<<< @@ -4380,7 +4436,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":888 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":888 * elif t == NPY_USHORT: f[0] = 72 #"H" * elif t == NPY_INT: f[0] = 105 #"i" * elif t == NPY_UINT: f[0] = 73 #"I" # <<<<<<<<<<<<<< @@ -4398,7 +4454,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":889 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":889 * elif t == NPY_INT: f[0] = 105 #"i" * elif t == NPY_UINT: f[0] = 73 #"I" * elif t == NPY_LONG: f[0] = 108 #"l" # <<<<<<<<<<<<<< @@ -4416,7 +4472,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":890 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":890 * elif t == NPY_UINT: f[0] = 73 #"I" * elif t == NPY_LONG: f[0] = 108 #"l" * elif t == NPY_ULONG: f[0] = 76 #"L" # <<<<<<<<<<<<<< @@ -4434,7 +4490,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":891 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":891 * elif t == NPY_LONG: f[0] = 108 #"l" * elif t == NPY_ULONG: f[0] = 76 #"L" * elif t == NPY_LONGLONG: f[0] = 113 #"q" # <<<<<<<<<<<<<< @@ -4452,7 +4508,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":892 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":892 * elif t == NPY_ULONG: f[0] = 76 #"L" * elif t == NPY_LONGLONG: f[0] = 113 #"q" * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" # <<<<<<<<<<<<<< @@ -4470,7 +4526,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":893 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":893 * elif t == NPY_LONGLONG: f[0] = 113 #"q" * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" * elif t == NPY_FLOAT: f[0] = 102 #"f" # <<<<<<<<<<<<<< @@ -4488,7 +4544,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":894 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":894 * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" * elif t == NPY_FLOAT: f[0] = 102 #"f" * elif t == NPY_DOUBLE: f[0] = 100 #"d" # <<<<<<<<<<<<<< @@ -4506,7 +4562,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":895 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":895 * elif t == NPY_FLOAT: f[0] = 102 #"f" * elif t == NPY_DOUBLE: f[0] = 100 #"d" * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" # <<<<<<<<<<<<<< @@ -4524,7 +4580,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":896 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":896 * elif t == NPY_DOUBLE: f[0] = 100 #"d" * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf # <<<<<<<<<<<<<< @@ -4544,7 +4600,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":897 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":897 * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd # <<<<<<<<<<<<<< @@ -4564,7 +4620,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":898 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":898 * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg # <<<<<<<<<<<<<< @@ -4584,7 +4640,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":899 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":899 * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg * elif t == NPY_OBJECT: f[0] = 79 #"O" # <<<<<<<<<<<<<< @@ -4602,7 +4658,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":901 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":901 * elif t == NPY_OBJECT: f[0] = 79 #"O" * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< @@ -4621,7 +4677,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L15:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":902 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":902 * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * f += 1 # <<<<<<<<<<<<<< @@ -4630,7 +4686,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_f = (__pyx_v_f + 1); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 * offset[0] += child.itemsize * * if not PyDataType_HASFIELDS(child): # <<<<<<<<<<<<<< @@ -4640,7 +4696,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L13; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":906 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":906 * # Cython ignores struct boundary information ("T{...}"), * # so don't output it * f = _util_dtypestring(child, f, end, offset) # <<<<<<<<<<<<<< @@ -4653,7 +4709,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L13:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 * cdef tuple fields * * for childname in descr.names: # <<<<<<<<<<<<<< @@ -4663,7 +4719,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":907 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":907 * # so don't output it * f = _util_dtypestring(child, f, end, offset) * return f # <<<<<<<<<<<<<< @@ -4673,7 +4729,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_r = __pyx_v_f; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 * return () * * cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: # <<<<<<<<<<<<<< @@ -4698,7 +4754,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 * int _import_umath() except -1 * * cdef inline void set_array_base(ndarray arr, object base): # <<<<<<<<<<<<<< @@ -4710,7 +4766,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("set_array_base", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1023 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1023 * * cdef inline void set_array_base(ndarray arr, object base): * Py_INCREF(base) # important to do this before stealing the reference below! # <<<<<<<<<<<<<< @@ -4719,7 +4775,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a */ Py_INCREF(__pyx_v_base); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1024 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1024 * cdef inline void set_array_base(ndarray arr, object base): * Py_INCREF(base) # important to do this before stealing the reference below! * PyArray_SetBaseObject(arr, base) # <<<<<<<<<<<<<< @@ -4728,7 +4784,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a */ (void)(PyArray_SetBaseObject(__pyx_v_arr, __pyx_v_base)); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 * int _import_umath() except -1 * * cdef inline void set_array_base(ndarray arr, object base): # <<<<<<<<<<<<<< @@ -4740,7 +4796,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a __Pyx_RefNannyFinishContext(); } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 * PyArray_SetBaseObject(arr, base) * * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< @@ -4755,7 +4811,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py int __pyx_t_1; __Pyx_RefNannySetupContext("get_array_base", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1027 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1027 * * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) # <<<<<<<<<<<<<< @@ -4764,7 +4820,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py */ __pyx_v_base = PyArray_BASE(__pyx_v_arr); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) * if base is NULL: # <<<<<<<<<<<<<< @@ -4774,7 +4830,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_t_1 = ((__pyx_v_base == NULL) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1029 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1029 * base = PyArray_BASE(arr) * if base is NULL: * return None # <<<<<<<<<<<<<< @@ -4785,7 +4841,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) * if base is NULL: # <<<<<<<<<<<<<< @@ -4794,7 +4850,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1030 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1030 * if base is NULL: * return None * return base # <<<<<<<<<<<<<< @@ -4806,7 +4862,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_r = ((PyObject *)__pyx_v_base); goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 * PyArray_SetBaseObject(arr, base) * * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< @@ -4821,7 +4877,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 * # Versions of the import_* functions which are more suitable for * # Cython code. * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< @@ -4842,7 +4898,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_array", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -4858,7 +4914,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1036 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1036 * cdef inline int import_array() except -1: * try: * _import_array() # <<<<<<<<<<<<<< @@ -4867,7 +4923,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { */ __pyx_t_4 = _import_array(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1036, __pyx_L3_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -4881,7 +4937,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1037 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1037 * try: * _import_array() * except Exception: # <<<<<<<<<<<<<< @@ -4896,7 +4952,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 * _import_array() * except Exception: * raise ImportError("numpy.core.multiarray failed to import") # <<<<<<<<<<<<<< @@ -4912,7 +4968,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -4927,7 +4983,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __pyx_L8_try_end:; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 * # Versions of the import_* functions which are more suitable for * # Cython code. * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< @@ -4950,7 +5006,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 * raise ImportError("numpy.core.multiarray failed to import") * * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< @@ -4971,7 +5027,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_umath", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -4987,7 +5043,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1042 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1042 * cdef inline int import_umath() except -1: * try: * _import_umath() # <<<<<<<<<<<<<< @@ -4996,7 +5052,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { */ __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1042, __pyx_L3_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -5010,7 +5066,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1043 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1043 * try: * _import_umath() * except Exception: # <<<<<<<<<<<<<< @@ -5025,7 +5081,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -5041,7 +5097,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -5056,7 +5112,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __pyx_L8_try_end:; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 * raise ImportError("numpy.core.multiarray failed to import") * * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< @@ -5079,7 +5135,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 * raise ImportError("numpy.core.umath failed to import") * * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< @@ -5100,7 +5156,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_ufunc", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5116,7 +5172,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1048 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1048 * cdef inline int import_ufunc() except -1: * try: * _import_umath() # <<<<<<<<<<<<<< @@ -5125,7 +5181,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { */ __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1048, __pyx_L3_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5139,7 +5195,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1049 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1049 * try: * _import_umath() * except Exception: # <<<<<<<<<<<<<< @@ -5153,7 +5209,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1050 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1050 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -5167,7 +5223,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5182,7 +5238,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __pyx_L8_try_end:; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 * raise ImportError("numpy.core.umath failed to import") * * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< @@ -5205,241 +5261,24 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { return __pyx_r; } -/* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /*proto*/ -static char __pyx_doc_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap[] = "wrap(b)"; -static PyMethodDef __pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap = {"wrap", (PyCFunction)__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, METH_O, __pyx_doc_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap}; -static PyObject *__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap(PyObject *__pyx_self, PyObject *__pyx_v_b) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("wrap (wrapper)", 0); - __pyx_r = __pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(__pyx_self, ((PyObject *)__pyx_v_b)); +static PyMethodDef __pyx_methods[] = { + {"ft_hash_bytes", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash_bytes}, + {"ft_hash_broken", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_broken}, + {"compute_ngrams", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_4compute_ngrams}, + {"compute_ngrams_bytes", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes}, + {0, 0, 0, 0} +}; - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b) { - struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_cur_scope; - struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_outer_scope; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("wrap", 0); - __pyx_outer_scope = (struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *) __Pyx_CyFunction_GetClosure(__pyx_self); - __pyx_cur_scope = __pyx_outer_scope; - - /* "cfunc.to_py":67 - * def wrap(object b): - * """wrap(b)""" - * return f(b) # <<<<<<<<<<<<<< - * return wrap - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_cur_scope->__pyx_v_f(__pyx_v_b); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 67, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("cfunc.to_py.__Pyx_CFunc_object____object___to_py.wrap", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "cfunc.to_py":64 - * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" - */ - -static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyObject *)) { - struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_cur_scope; - PyObject *__pyx_v_wrap = 0; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__Pyx_CFunc_object____object___to_py", 0); - __pyx_cur_scope = (struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __pyx_cur_scope = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)Py_None); - __Pyx_INCREF(Py_None); - __PYX_ERR(2, 64, __pyx_L1_error) - } else { - __Pyx_GOTREF(__pyx_cur_scope); - } - __pyx_cur_scope->__pyx_v_f = __pyx_v_f; - - /* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_wrap = __pyx_t_1; - __pyx_t_1 = 0; - - /* "cfunc.to_py":68 - * """wrap(b)""" - * return f(b) - * return wrap # <<<<<<<<<<<<<< - * - * - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_wrap); - __pyx_r = __pyx_v_wrap; - goto __pyx_L0; - - /* "cfunc.to_py":64 - * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("cfunc.to_py.__Pyx_CFunc_object____object___to_py", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_wrap); - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[8]; -static int __pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; - -static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { - o = (PyObject*)__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[--__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py]; - memset(o, 0, sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)); - (void) PyObject_INIT(o, t); - } else { - o = (*t->tp_alloc)(t, 0); - if (unlikely(!o)) return 0; - } - return o; -} - -static void __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyObject *o) { - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { - __pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py++] = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)o); - } else { - (*Py_TYPE(o)->tp_free)(o); - } -} - -static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { - PyVarObject_HEAD_INIT(0, 0) - "gensim.models._utils_any2vec.__pyx_scope_struct____Pyx_CFunc_object____object___to_py", /*tp_name*/ - sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyMethodDef __pyx_methods[] = { - {"ft_hash", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash}, - {"ft_hash_broken", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_broken}, - {"compute_ngrams", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_4compute_ngrams}, - {0, 0, 0, 0} -}; - -#if PY_MAJOR_VERSION >= 3 -#if CYTHON_PEP489_MULTI_PHASE_INIT -static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ -static int __pyx_pymod_exec__utils_any2vec(PyObject* module); /*proto*/ -static PyModuleDef_Slot __pyx_moduledef_slots[] = { - {Py_mod_create, (void*)__pyx_pymod_create}, - {Py_mod_exec, (void*)__pyx_pymod_exec__utils_any2vec}, - {0, NULL} -}; -#endif +#if PY_MAJOR_VERSION >= 3 +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec__utils_any2vec(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec__utils_any2vec}, + {0, NULL} +}; +#endif static struct PyModuleDef __pyx_moduledef = { PyModuleDef_HEAD_INIT, @@ -5478,16 +5317,12 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_ImportError, __pyx_k_ImportError, sizeof(__pyx_k_ImportError), 0, 0, 1, 1}, {&__pyx_kp_u_Non_native_byte_order_not_suppor, __pyx_k_Non_native_byte_order_not_suppor, sizeof(__pyx_k_Non_native_byte_order_not_suppor), 0, 1, 0, 0}, {&__pyx_n_s_PY2, __pyx_k_PY2, sizeof(__pyx_k_PY2), 0, 0, 1, 1}, - {&__pyx_n_s_Pyx_CFunc_object____object___t, __pyx_k_Pyx_CFunc_object____object___t, sizeof(__pyx_k_Pyx_CFunc_object____object___t), 0, 0, 1, 1}, {&__pyx_n_s_RuntimeError, __pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 0, 1, 1}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, {&__pyx_kp_u__2, __pyx_k__2, sizeof(__pyx_k__2), 0, 1, 0, 0}, - {&__pyx_n_s_b, __pyx_k_b, sizeof(__pyx_k_b), 0, 0, 1, 1}, - {&__pyx_n_s_byte_to_int, __pyx_k_byte_to_int, sizeof(__pyx_k_byte_to_int), 0, 0, 1, 1}, - {&__pyx_n_s_cfunc_to_py, __pyx_k_cfunc_to_py, sizeof(__pyx_k_cfunc_to_py), 0, 0, 1, 1}, {&__pyx_n_s_cline_in_traceback, __pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 0, 1, 1}, + {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, - {&__pyx_n_s_int8, __pyx_k_int8, sizeof(__pyx_k_int8), 0, 0, 1, 1}, {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, {&__pyx_n_s_max_n, __pyx_k_max_n, sizeof(__pyx_k_max_n), 0, 0, 1, 1}, {&__pyx_n_s_min_n, __pyx_k_min_n, sizeof(__pyx_k_min_n), 0, 0, 1, 1}, @@ -5499,13 +5334,12 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_s_numpy_core_multiarray_failed_to, __pyx_k_numpy_core_multiarray_failed_to, sizeof(__pyx_k_numpy_core_multiarray_failed_to), 0, 0, 1, 0}, {&__pyx_kp_s_numpy_core_umath_failed_to_impor, __pyx_k_numpy_core_umath_failed_to_impor, sizeof(__pyx_k_numpy_core_umath_failed_to_impor), 0, 0, 1, 0}, {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, + {&__pyx_kp_s_s, __pyx_k_s, sizeof(__pyx_k_s), 0, 0, 1, 0}, {&__pyx_n_s_six, __pyx_k_six, sizeof(__pyx_k_six), 0, 0, 1, 1}, - {&__pyx_kp_s_stringsource, __pyx_k_stringsource, sizeof(__pyx_k_stringsource), 0, 0, 1, 0}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, - {&__pyx_n_s_uint32, __pyx_k_uint32, sizeof(__pyx_k_uint32), 0, 0, 1, 1}, {&__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_k_unknown_dtype_code_in_numpy_pxd, sizeof(__pyx_k_unknown_dtype_code_in_numpy_pxd), 0, 1, 0, 0}, + {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, {&__pyx_n_s_word, __pyx_k_word, sizeof(__pyx_k_word), 0, 0, 1, 1}, - {&__pyx_n_s_wrap, __pyx_k_wrap, sizeof(__pyx_k_wrap), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) { @@ -5522,7 +5356,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< @@ -5533,7 +5367,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__3); __Pyx_GIVEREF(__pyx_tuple__3); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< @@ -5544,7 +5378,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__4); __Pyx_GIVEREF(__pyx_tuple__4); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -5555,7 +5389,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__5); __Pyx_GIVEREF(__pyx_tuple__5); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 * * if (end - f) - (new_offset - offset[0]) < 15: * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") # <<<<<<<<<<<<<< @@ -5566,7 +5400,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__6); __Pyx_GIVEREF(__pyx_tuple__6); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 * t = child.type_num * if end - f < 5: * raise RuntimeError(u"Format string allocated too short.") # <<<<<<<<<<<<<< @@ -5577,7 +5411,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__7); __Pyx_GIVEREF(__pyx_tuple__7); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 * _import_array() * except Exception: * raise ImportError("numpy.core.multiarray failed to import") # <<<<<<<<<<<<<< @@ -5588,7 +5422,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__8); __Pyx_GIVEREF(__pyx_tuple__8); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -5598,18 +5432,6 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_s_numpy_core_umath_failed_to_impor); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(1, 1044, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__9); __Pyx_GIVEREF(__pyx_tuple__9); - - /* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - __pyx_tuple__10 = PyTuple_Pack(1, __pyx_n_s_b); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__10); - __Pyx_GIVEREF(__pyx_tuple__10); - __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 65, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -5621,7 +5443,6 @@ static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) { if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); __pyx_int_0 = PyInt_FromLong(0); if (unlikely(!__pyx_int_0)) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_int_1 = PyInt_FromLong(1); if (unlikely(!__pyx_int_1)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_int_16777619 = PyInt_FromLong(16777619L); if (unlikely(!__pyx_int_16777619)) __PYX_ERR(0, 1, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -5663,17 +5484,8 @@ static int __Pyx_modinit_type_init_code(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) - __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; - if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_dictoffset && __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_getattro == PyObject_GenericGetAttr)) { - __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; - } - __pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = &__pyx_scope_struct____Pyx_CFunc_object____object___to_py; __Pyx_RefNannyFinishContext(); return 0; - __pyx_L1_error:; - __Pyx_RefNannyFinishContext(); - return -1; } static int __Pyx_modinit_type_import_code(void) { @@ -5681,7 +5493,7 @@ static int __Pyx_modinit_type_import_code(void) { PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); /*--- Type import code ---*/ - __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 9, __pyx_L1_error) + __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 9, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_ptype_7cpython_4type_type = __Pyx_ImportType(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "type", #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 @@ -5690,7 +5502,7 @@ static int __Pyx_modinit_type_import_code(void) { sizeof(PyHeapTypeObject), #endif __Pyx_ImportType_CheckSize_Warn); - if (!__pyx_ptype_7cpython_4type_type) __PYX_ERR(3, 9, __pyx_L1_error) + if (!__pyx_ptype_7cpython_4type_type) __PYX_ERR(2, 9, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyImport_ImportModule("numpy"); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 206, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); @@ -5823,7 +5635,6 @@ static CYTHON_SMALL_CODE int __pyx_pymod_exec__utils_any2vec(PyObject *__pyx_pyi { PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; - int __pyx_t_3; __Pyx_RefNannyDeclarations #if CYTHON_PEP489_MULTI_PHASE_INIT if (__pyx_m) { @@ -5920,7 +5731,7 @@ if (!__Pyx_RefNanny) { (void)__Pyx_modinit_global_init_code(); (void)__Pyx_modinit_variable_export_code(); (void)__Pyx_modinit_function_export_code(); - if (unlikely(__Pyx_modinit_type_init_code() != 0)) goto __pyx_L1_error; + (void)__Pyx_modinit_type_init_code(); if (unlikely(__Pyx_modinit_type_import_code() != 0)) goto __pyx_L1_error; (void)__Pyx_modinit_variable_import_code(); (void)__Pyx_modinit_function_import_code(); @@ -5929,63 +5740,56 @@ if (!__Pyx_RefNanny) { if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif - /* "gensim/models/_utils_any2vec.pyx":10 - * """General functions used for any2vec models.""" + /* "gensim/models/_utils_any2vec.pyx":17 + * ctypedef signed char int8_t * * from six import PY2 # <<<<<<<<<<<<<< * import numpy as np * cimport numpy as np */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 10, __pyx_L1_error) + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 17, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_n_s_PY2); __Pyx_GIVEREF(__pyx_n_s_PY2); PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_PY2); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_six, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_six, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 17, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 10, __pyx_L1_error) + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 17, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_PY2, __pyx_t_1) < 0) __PYX_ERR(0, 10, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_PY2, __pyx_t_1) < 0) __PYX_ERR(0, 17, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "gensim/models/_utils_any2vec.pyx":11 + /* "gensim/models/_utils_any2vec.pyx":18 * * from six import PY2 * import numpy as np # <<<<<<<<<<<<<< * cimport numpy as np * */ - __pyx_t_2 = __Pyx_Import(__pyx_n_s_numpy, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 11, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_numpy, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 18, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_2) < 0) __PYX_ERR(0, 11, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_2) < 0) __PYX_ERR(0, 18, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "gensim/models/_utils_any2vec.pyx":21 - * return ord(b) + /* "gensim/models/_utils_any2vec.pyx":100 + * # as opposed to a new character. + * # + * cdef unsigned char _MB_MASK = 0xC0 # <<<<<<<<<<<<<< + * cdef unsigned char _MB_START = 0x80 * - * _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 # <<<<<<<<<<<<<< + */ + __pyx_v_6gensim_6models_14_utils_any2vec__MB_MASK = 0xC0; + + /* "gensim/models/_utils_any2vec.pyx":101 + * # + * cdef unsigned char _MB_MASK = 0xC0 + * cdef unsigned char _MB_START = 0x80 # <<<<<<<<<<<<<< * * */ - __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 21, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (__pyx_t_3) { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_t_1; - __pyx_t_1 = 0; - } else { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_t_1; - __pyx_t_1 = 0; - } - if (PyDict_SetItem(__pyx_d, __pyx_n_s_byte_to_int, __pyx_t_2) < 0) __PYX_ERR(0, 21, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_6gensim_6models_14_utils_any2vec__MB_START = 0x80; /* "gensim/models/_utils_any2vec.pyx":1 * #!/usr/bin/env cython # <<<<<<<<<<<<<< @@ -5997,12 +5801,12 @@ if (!__Pyx_RefNanny) { if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_2) < 0) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "cfunc.to_py":64 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 + * raise ImportError("numpy.core.umath failed to import") * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" + * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< + * try: + * _import_umath() */ /*--- Wrapped vars code ---*/ @@ -6076,6 +5880,43 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name) { return result; } +/* ArgTypeTest */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (exact) { + #if PY_MAJOR_VERSION == 2 + if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; + #endif + } + else { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", + name, type->tp_name, Py_TYPE(obj)->tp_name); + return 0; +} + +/* unicode_iter */ +static CYTHON_INLINE int __Pyx_init_unicode_iteration( + PyObject* ustring, Py_ssize_t *length, void** data, int *kind) { +#if CYTHON_PEP393_ENABLED + if (unlikely(__Pyx_PyUnicode_READY(ustring) < 0)) return -1; + *kind = PyUnicode_KIND(ustring); + *length = PyUnicode_GET_LENGTH(ustring); + *data = PyUnicode_DATA(ustring); +#else + *kind = 0; + *length = PyUnicode_GET_SIZE(ustring); + *data = (void*)PyUnicode_AS_UNICODE(ustring); +#endif + return 0; +} + /* UnicodeAsUCS4 */ static CYTHON_INLINE Py_UCS4 __Pyx_PyUnicode_AsPy_UCS4(PyObject* x) { Py_ssize_t length; @@ -6136,391 +5977,88 @@ static long __Pyx__PyObject_Ord(PyObject* c) { return (long)(Py_UCS4)-1; } -/* GetModuleGlobalName */ -#if CYTHON_USE_DICT_VERSIONS -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +/* JoinPyUnicode */ +static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_count, Py_ssize_t result_ulength, + CYTHON_UNUSED Py_UCS4 max_char) { +#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + PyObject *result_uval; + int result_ukind; + Py_ssize_t i, char_pos; + void *result_udata; +#if CYTHON_PEP393_ENABLED + result_uval = PyUnicode_New(result_ulength, max_char); + if (unlikely(!result_uval)) return NULL; + result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + result_udata = PyUnicode_DATA(result_uval); #else -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) + result_uval = PyUnicode_FromUnicode(NULL, result_ulength); + if (unlikely(!result_uval)) return NULL; + result_ukind = sizeof(Py_UNICODE); + result_udata = PyUnicode_AS_UNICODE(result_uval); #endif -{ - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 - result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } else if (unlikely(PyErr_Occurred())) { - return NULL; - } -#else - result = PyDict_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); + char_pos = 0; + for (i=0; i < value_count; i++) { + int ukind; + Py_ssize_t ulength; + void *udata; + PyObject *uval = PyTuple_GET_ITEM(value_tuple, i); + if (unlikely(__Pyx_PyUnicode_READY(uval))) + goto bad; + ulength = __Pyx_PyUnicode_GET_LENGTH(uval); + if (unlikely(!ulength)) + continue; + if (unlikely(char_pos + ulength < 0)) + goto overflow; + ukind = __Pyx_PyUnicode_KIND(uval); + udata = __Pyx_PyUnicode_DATA(uval); + if (!CYTHON_PEP393_ENABLED || ukind == result_ukind) { + memcpy((char *)result_udata + char_pos * result_ukind, udata, (size_t) (ulength * result_ukind)); + } else { + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030300F0 || defined(_PyUnicode_FastCopyCharacters) + _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); + #else + Py_ssize_t j; + for (j=0; j < ulength; j++) { + Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j); + __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar); + } + #endif + } + char_pos += ulength; } -#endif + return result_uval; +overflow: + PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string"); +bad: + Py_DECREF(result_uval); + return NULL; #else - result = PyObject_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } - PyErr_Clear(); + result_ulength++; + value_count++; + return PyUnicode_Join(__pyx_empty_unicode, value_tuple); #endif - return __Pyx_GetBuiltinName(name); } -/* PyCFunctionFastCall */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - int flags = PyCFunction_GET_FLAGS(func); - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { - return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); - } else { - return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = func->ob_type->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); } + return result; } #endif -/* PyFunctionFastCall */ -#if CYTHON_FAST_PYCALL -static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, - PyObject *globals) { - PyFrameObject *f; - PyThreadState *tstate = __Pyx_PyThreadState_Current; - PyObject **fastlocals; - Py_ssize_t i; - PyObject *result; - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) { - return NULL; - } - fastlocals = __Pyx_PyFrame_GetLocalsplus(f); - for (i = 0; i < na; i++) { - Py_INCREF(*args); - fastlocals[i] = *args++; - } - result = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return result; -} -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *closure; -#if PY_MAJOR_VERSION >= 3 - PyObject *kwdefs; -#endif - PyObject *kwtuple, **k; - PyObject **d; - Py_ssize_t nd; - Py_ssize_t nk; - PyObject *result; - assert(kwargs == NULL || PyDict_Check(kwargs)); - nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { - return NULL; - } - if ( -#if PY_MAJOR_VERSION >= 3 - co->co_kwonlyargcount == 0 && -#endif - likely(kwargs == NULL || nk == 0) && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - if (argdefs == NULL && co->co_argcount == nargs) { - result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); - goto done; - } - else if (nargs == 0 && argdefs != NULL - && co->co_argcount == Py_SIZE(argdefs)) { - /* function called with no arguments, but all parameters have - a default value: use default values as arguments .*/ - args = &PyTuple_GET_ITEM(argdefs, 0); - result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); - goto done; - } - } - if (kwargs != NULL) { - Py_ssize_t pos, i; - kwtuple = PyTuple_New(2 * nk); - if (kwtuple == NULL) { - result = NULL; - goto done; - } - k = &PyTuple_GET_ITEM(kwtuple, 0); - pos = i = 0; - while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { - Py_INCREF(k[i]); - Py_INCREF(k[i+1]); - i += 2; - } - nk = i / 2; - } - else { - kwtuple = NULL; - k = NULL; - } - closure = PyFunction_GET_CLOSURE(func); -#if PY_MAJOR_VERSION >= 3 - kwdefs = PyFunction_GET_KW_DEFAULTS(func); -#endif - if (argdefs != NULL) { - d = &PyTuple_GET_ITEM(argdefs, 0); - nd = Py_SIZE(argdefs); - } - else { - d = NULL; - nd = 0; - } -#if PY_MAJOR_VERSION >= 3 - result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, kwdefs, closure); -#else - result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, closure); -#endif - Py_XDECREF(kwtuple); -done: - Py_LeaveRecursiveCall(); - return result; -} -#endif -#endif - -/* PyObjectCall */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { - PyObject *result; - ternaryfunc call = func->ob_type->tp_call; - if (unlikely(!call)) - return PyObject_Call(func, arg, kw); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = (*call)(func, arg, kw); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyObjectCall2Args */ -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { - PyObject *args, *result = NULL; - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyFunction_FastCall(function, args, 2); - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyCFunction_FastCall(function, args, 2); - } - #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 0, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 1, arg2); - Py_INCREF(function); - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); -done: - return result; -} - -/* PyObjectCallMethO */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { - PyObject *self, *result; - PyCFunction cfunc; - cfunc = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = cfunc(self, arg); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyObjectCallOneArg */ -#if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif - if (likely(PyCFunction_Check(func))) { - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif - } - } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -#endif - -/* ArgTypeTest */ -static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) -{ - if (unlikely(!type)) { - PyErr_SetString(PyExc_SystemError, "Missing type object"); - return 0; - } - else if (exact) { - #if PY_MAJOR_VERSION == 2 - if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; - #endif - } - else { - if (likely(__Pyx_TypeCheck(obj, type))) return 1; - } - PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); - return 0; -} - -/* unicode_iter */ -static CYTHON_INLINE int __Pyx_init_unicode_iteration( - PyObject* ustring, Py_ssize_t *length, void** data, int *kind) { -#if CYTHON_PEP393_ENABLED - if (unlikely(__Pyx_PyUnicode_READY(ustring) < 0)) return -1; - *kind = PyUnicode_KIND(ustring); - *length = PyUnicode_GET_LENGTH(ustring); - *data = PyUnicode_DATA(ustring); -#else - *kind = 0; - *length = PyUnicode_GET_SIZE(ustring); - *data = (void*)PyUnicode_AS_UNICODE(ustring); -#endif - return 0; -} - -/* JoinPyUnicode */ -static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_count, Py_ssize_t result_ulength, - CYTHON_UNUSED Py_UCS4 max_char) { -#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - PyObject *result_uval; - int result_ukind; - Py_ssize_t i, char_pos; - void *result_udata; -#if CYTHON_PEP393_ENABLED - result_uval = PyUnicode_New(result_ulength, max_char); - if (unlikely(!result_uval)) return NULL; - result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; - result_udata = PyUnicode_DATA(result_uval); -#else - result_uval = PyUnicode_FromUnicode(NULL, result_ulength); - if (unlikely(!result_uval)) return NULL; - result_ukind = sizeof(Py_UNICODE); - result_udata = PyUnicode_AS_UNICODE(result_uval); -#endif - char_pos = 0; - for (i=0; i < value_count; i++) { - int ukind; - Py_ssize_t ulength; - void *udata; - PyObject *uval = PyTuple_GET_ITEM(value_tuple, i); - if (unlikely(__Pyx_PyUnicode_READY(uval))) - goto bad; - ulength = __Pyx_PyUnicode_GET_LENGTH(uval); - if (unlikely(!ulength)) - continue; - if (unlikely(char_pos + ulength < 0)) - goto overflow; - ukind = __Pyx_PyUnicode_KIND(uval); - udata = __Pyx_PyUnicode_DATA(uval); - if (!CYTHON_PEP393_ENABLED || ukind == result_ukind) { - memcpy((char *)result_udata + char_pos * result_ukind, udata, (size_t) (ulength * result_ukind)); - } else { - #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030300F0 || defined(_PyUnicode_FastCopyCharacters) - _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); - #else - Py_ssize_t j; - for (j=0; j < ulength; j++) { - Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j); - __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar); - } - #endif - } - char_pos += ulength; - } - return result_uval; -overflow: - PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string"); -bad: - Py_DECREF(result_uval); - return NULL; -#else - result_ulength++; - value_count++; - return PyUnicode_Join(__pyx_empty_unicode, value_tuple); -#endif -} - /* PyIntBinop */ #if !CYTHON_COMPILING_IN_PYPY static PyObject* __Pyx_PyInt_AddObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { @@ -6790,26 +6328,257 @@ static int __Pyx_ParseOptionalKeywords( goto invalid_keyword; } } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, key); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%.200s() got an unexpected keyword argument '%.200s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%.200s() got an unexpected keyword argument '%.200s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +/* PyCFunctionFastCall */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + int flags = PyCFunction_GET_FLAGS(func); + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { + return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); + } else { + return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); + } +} +#endif + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; + } + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && +#endif + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); +#endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); +#else + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); +#endif + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif +#endif + +/* PyObjectCall2Args */ +static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args, *result = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(function)) { + PyObject *args[2] = {arg1, arg2}; + return __Pyx_PyFunction_FastCall(function, args, 2); + } + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(function)) { + PyObject *args[2] = {arg1, arg2}; + return __Pyx_PyCFunction_FastCall(function, args, 2); + } + #endif + args = PyTuple_New(2); + if (unlikely(!args)) goto done; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + Py_INCREF(function); + result = __Pyx_PyObject_Call(function, args, NULL); + Py_DECREF(args); + Py_DECREF(function); +done: + return result; +} + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallOneArg */ +#if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); + } +#endif + if (likely(PyCFunction_Check(func))) { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif + } + } + return __Pyx__PyObject_CallOneArg(func, arg); } +#else +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +#endif /* PyErrFetchRestore */ #if CYTHON_FAST_THREAD_STATE @@ -7173,711 +6942,36 @@ static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { _PyErr_StackItem *exc_info = tstate->exc_info; tmp_type = exc_info->exc_type; - tmp_value = exc_info->exc_value; - tmp_tb = exc_info->exc_traceback; - exc_info->exc_type = local_type; - exc_info->exc_value = local_value; - exc_info->exc_traceback = local_tb; - } - #else - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = local_type; - tstate->exc_value = local_value; - tstate->exc_traceback = local_tb; - #endif - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -#else - PyErr_SetExcInfo(local_type, local_value, local_tb); -#endif - return 0; -bad: - *type = 0; - *value = 0; - *tb = 0; - Py_XDECREF(local_type); - Py_XDECREF(local_value); - Py_XDECREF(local_tb); - return -1; -} - -/* FetchCommonType */ -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); - if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); - goto bad; - } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); - goto bad; - } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; - } -done: - Py_DECREF(fake_module); - return cached_type; -bad: - Py_XDECREF(cached_type); - cached_type = NULL; - goto done; -} - -/* CythonFunction */ -#include -static PyObject * -__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) -{ - if (unlikely(op->func_doc == NULL)) { - if (op->func.m_ml->ml_doc) { -#if PY_MAJOR_VERSION >= 3 - op->func_doc = PyUnicode_FromString(op->func.m_ml->ml_doc); -#else - op->func_doc = PyString_FromString(op->func.m_ml->ml_doc); -#endif - if (unlikely(op->func_doc == NULL)) - return NULL; - } else { - Py_INCREF(Py_None); - return Py_None; - } - } - Py_INCREF(op->func_doc); - return op->func_doc; -} -static int -__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp = op->func_doc; - if (value == NULL) { - value = Py_None; - } - Py_INCREF(value); - op->func_doc = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - if (unlikely(op->func_name == NULL)) { -#if PY_MAJOR_VERSION >= 3 - op->func_name = PyUnicode_InternFromString(op->func.m_ml->ml_name); -#else - op->func_name = PyString_InternFromString(op->func.m_ml->ml_name); -#endif - if (unlikely(op->func_name == NULL)) - return NULL; - } - Py_INCREF(op->func_name); - return op->func_name; -} -static int -__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) -#else - if (unlikely(value == NULL || !PyString_Check(value))) -#endif - { - PyErr_SetString(PyExc_TypeError, - "__name__ must be set to a string object"); - return -1; - } - tmp = op->func_name; - Py_INCREF(value); - op->func_name = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - Py_INCREF(op->func_qualname); - return op->func_qualname; -} -static int -__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) -#else - if (unlikely(value == NULL || !PyString_Check(value))) -#endif - { - PyErr_SetString(PyExc_TypeError, - "__qualname__ must be set to a string object"); - return -1; - } - tmp = op->func_qualname; - Py_INCREF(value); - op->func_qualname = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) -{ - PyObject *self; - self = m->func_closure; - if (self == NULL) - self = Py_None; - Py_INCREF(self); - return self; -} -static PyObject * -__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - if (unlikely(op->func_dict == NULL)) { - op->func_dict = PyDict_New(); - if (unlikely(op->func_dict == NULL)) - return NULL; - } - Py_INCREF(op->func_dict); - return op->func_dict; -} -static int -__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp; - if (unlikely(value == NULL)) { - PyErr_SetString(PyExc_TypeError, - "function's dictionary may not be deleted"); - return -1; - } - if (unlikely(!PyDict_Check(value))) { - PyErr_SetString(PyExc_TypeError, - "setting function's dictionary to a non-dict"); - return -1; - } - tmp = op->func_dict; - Py_INCREF(value); - op->func_dict = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - Py_INCREF(op->func_globals); - return op->func_globals; -} -static PyObject * -__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - Py_INCREF(Py_None); - return Py_None; -} -static PyObject * -__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - PyObject* result = (op->func_code) ? op->func_code : Py_None; - Py_INCREF(result); - return result; -} -static int -__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { - int result = 0; - PyObject *res = op->defaults_getter((PyObject *) op); - if (unlikely(!res)) - return -1; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - op->defaults_tuple = PyTuple_GET_ITEM(res, 0); - Py_INCREF(op->defaults_tuple); - op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); - Py_INCREF(op->defaults_kwdict); - #else - op->defaults_tuple = PySequence_ITEM(res, 0); - if (unlikely(!op->defaults_tuple)) result = -1; - else { - op->defaults_kwdict = PySequence_ITEM(res, 1); - if (unlikely(!op->defaults_kwdict)) result = -1; - } - #endif - Py_DECREF(res); - return result; -} -static int -__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; - if (!value) { - value = Py_None; - } else if (value != Py_None && !PyTuple_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "__defaults__ must be set to a tuple object"); - return -1; - } - Py_INCREF(value); - tmp = op->defaults_tuple; - op->defaults_tuple = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { - PyObject* result = op->defaults_tuple; - if (unlikely(!result)) { - if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; - result = op->defaults_tuple; - } else { - result = Py_None; - } - } - Py_INCREF(result); - return result; -} -static int -__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; - if (!value) { - value = Py_None; - } else if (value != Py_None && !PyDict_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "__kwdefaults__ must be set to a dict object"); - return -1; - } - Py_INCREF(value); - tmp = op->defaults_kwdict; - op->defaults_kwdict = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { - PyObject* result = op->defaults_kwdict; - if (unlikely(!result)) { - if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; - result = op->defaults_kwdict; - } else { - result = Py_None; - } - } - Py_INCREF(result); - return result; -} -static int -__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; - if (!value || value == Py_None) { - value = NULL; - } else if (!PyDict_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "__annotations__ must be set to a dict object"); - return -1; - } - Py_XINCREF(value); - tmp = op->func_annotations; - op->func_annotations = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { - PyObject* result = op->func_annotations; - if (unlikely(!result)) { - result = PyDict_New(); - if (unlikely(!result)) return NULL; - op->func_annotations = result; - } - Py_INCREF(result); - return result; -} -static PyGetSetDef __pyx_CyFunction_getsets[] = { - {(char *) "func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, - {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, - {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, - {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, - {(char *) "__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, - {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, - {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, - {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, - {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, - {(char *) "__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, - {(char *) "func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, - {(char *) "__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, - {(char *) "func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, - {(char *) "__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, - {(char *) "func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, - {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, - {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, - {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, - {0, 0, 0, 0, 0} -}; -static PyMemberDef __pyx_CyFunction_members[] = { - {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), PY_WRITE_RESTRICTED, 0}, - {0, 0, 0, 0, 0} -}; -static PyObject * -__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) -{ -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_FromString(m->func.m_ml->ml_name); -#else - return PyString_FromString(m->func.m_ml->ml_name); -#endif -} -static PyMethodDef __pyx_CyFunction_methods[] = { - {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, - {0, 0, 0, 0} -}; -#if PY_VERSION_HEX < 0x030500A0 -#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) -#else -#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func.m_weakreflist) -#endif -static PyObject *__Pyx_CyFunction_New(PyTypeObject *type, PyMethodDef *ml, int flags, PyObject* qualname, - PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { - __pyx_CyFunctionObject *op = PyObject_GC_New(__pyx_CyFunctionObject, type); - if (op == NULL) - return NULL; - op->flags = flags; - __Pyx_CyFunction_weakreflist(op) = NULL; - op->func.m_ml = ml; - op->func.m_self = (PyObject *) op; - Py_XINCREF(closure); - op->func_closure = closure; - Py_XINCREF(module); - op->func.m_module = module; - op->func_dict = NULL; - op->func_name = NULL; - Py_INCREF(qualname); - op->func_qualname = qualname; - op->func_doc = NULL; - op->func_classobj = NULL; - op->func_globals = globals; - Py_INCREF(op->func_globals); - Py_XINCREF(code); - op->func_code = code; - op->defaults_pyobjects = 0; - op->defaults = NULL; - op->defaults_tuple = NULL; - op->defaults_kwdict = NULL; - op->defaults_getter = NULL; - op->func_annotations = NULL; - PyObject_GC_Track(op); - return (PyObject *) op; -} -static int -__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) -{ - Py_CLEAR(m->func_closure); - Py_CLEAR(m->func.m_module); - Py_CLEAR(m->func_dict); - Py_CLEAR(m->func_name); - Py_CLEAR(m->func_qualname); - Py_CLEAR(m->func_doc); - Py_CLEAR(m->func_globals); - Py_CLEAR(m->func_code); - Py_CLEAR(m->func_classobj); - Py_CLEAR(m->defaults_tuple); - Py_CLEAR(m->defaults_kwdict); - Py_CLEAR(m->func_annotations); - if (m->defaults) { - PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); - int i; - for (i = 0; i < m->defaults_pyobjects; i++) - Py_XDECREF(pydefaults[i]); - PyObject_Free(m->defaults); - m->defaults = NULL; - } - return 0; -} -static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) -{ - if (__Pyx_CyFunction_weakreflist(m) != NULL) - PyObject_ClearWeakRefs((PyObject *) m); - __Pyx_CyFunction_clear(m); - PyObject_GC_Del(m); -} -static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) -{ - PyObject_GC_UnTrack(m); - __Pyx__CyFunction_dealloc(m); -} -static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) -{ - Py_VISIT(m->func_closure); - Py_VISIT(m->func.m_module); - Py_VISIT(m->func_dict); - Py_VISIT(m->func_name); - Py_VISIT(m->func_qualname); - Py_VISIT(m->func_doc); - Py_VISIT(m->func_globals); - Py_VISIT(m->func_code); - Py_VISIT(m->func_classobj); - Py_VISIT(m->defaults_tuple); - Py_VISIT(m->defaults_kwdict); - if (m->defaults) { - PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); - int i; - for (i = 0; i < m->defaults_pyobjects; i++) - Py_VISIT(pydefaults[i]); - } - return 0; -} -static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) -{ - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { - Py_INCREF(func); - return func; - } - if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { - if (type == NULL) - type = (PyObject *)(Py_TYPE(obj)); - return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type))); - } - if (obj == Py_None) - obj = NULL; - return __Pyx_PyMethod_New(func, obj, type); -} -static PyObject* -__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) -{ -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_FromFormat("", - op->func_qualname, (void *)op); -#else - return PyString_FromFormat("", - PyString_AsString(op->func_qualname), (void *)op); -#endif -} -static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { - PyCFunctionObject* f = (PyCFunctionObject*)func; - PyCFunction meth = f->m_ml->ml_meth; - Py_ssize_t size; - switch (f->m_ml->ml_flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { - case METH_VARARGS: - if (likely(kw == NULL || PyDict_Size(kw) == 0)) - return (*meth)(self, arg); - break; - case METH_VARARGS | METH_KEYWORDS: - return (*(PyCFunctionWithKeywords)(void*)meth)(self, arg, kw); - case METH_NOARGS: - if (likely(kw == NULL || PyDict_Size(kw) == 0)) { - size = PyTuple_GET_SIZE(arg); - if (likely(size == 0)) - return (*meth)(self, NULL); - PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)", - f->m_ml->ml_name, size); - return NULL; - } - break; - case METH_O: - if (likely(kw == NULL || PyDict_Size(kw) == 0)) { - size = PyTuple_GET_SIZE(arg); - if (likely(size == 1)) { - PyObject *result, *arg0; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - arg0 = PyTuple_GET_ITEM(arg, 0); - #else - arg0 = PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; - #endif - result = (*meth)(self, arg0); - #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_DECREF(arg0); - #endif - return result; - } - PyErr_Format(PyExc_TypeError, - "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)", - f->m_ml->ml_name, size); - return NULL; - } - break; - default: - PyErr_SetString(PyExc_SystemError, "Bad call flags in " - "__Pyx_CyFunction_Call. METH_OLDARGS is no " - "longer supported!"); - return NULL; - } - PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", - f->m_ml->ml_name); - return NULL; -} -static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { - return __Pyx_CyFunction_CallMethod(func, ((PyCFunctionObject*)func)->m_self, arg, kw); -} -static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { - PyObject *result; - __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; - if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { - Py_ssize_t argc; - PyObject *new_args; - PyObject *self; - argc = PyTuple_GET_SIZE(args); - new_args = PyTuple_GetSlice(args, 1, argc); - if (unlikely(!new_args)) - return NULL; - self = PyTuple_GetItem(args, 0); - if (unlikely(!self)) { - Py_DECREF(new_args); - return NULL; - } - result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); - Py_DECREF(new_args); - } else { - result = __Pyx_CyFunction_Call(func, args, kw); - } - return result; -} -static PyTypeObject __pyx_CyFunctionType_type = { - PyVarObject_HEAD_INIT(0, 0) - "cython_function_or_method", - sizeof(__pyx_CyFunctionObject), - 0, - (destructor) __Pyx_CyFunction_dealloc, - 0, - 0, - 0, -#if PY_MAJOR_VERSION < 3 - 0, -#else - 0, -#endif - (reprfunc) __Pyx_CyFunction_repr, - 0, - 0, - 0, - 0, - __Pyx_CyFunction_CallAsMethod, - 0, - 0, - 0, - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - 0, - (traverseproc) __Pyx_CyFunction_traverse, - (inquiry) __Pyx_CyFunction_clear, - 0, -#if PY_VERSION_HEX < 0x030500A0 - offsetof(__pyx_CyFunctionObject, func_weakreflist), -#else - offsetof(PyCFunctionObject, m_weakreflist), -#endif - 0, - 0, - __pyx_CyFunction_methods, - __pyx_CyFunction_members, - __pyx_CyFunction_getsets, - 0, - 0, - __Pyx_CyFunction_descr_get, - 0, - offsetof(__pyx_CyFunctionObject, func_dict), - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, -#if PY_VERSION_HEX >= 0x030400a1 - 0, -#endif -}; -static int __pyx_CyFunction_init(void) { - __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); - if (unlikely(__pyx_CyFunctionType == NULL)) { - return -1; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; } - return 0; -} -static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->defaults = PyObject_Malloc(size); - if (unlikely(!m->defaults)) - return PyErr_NoMemory(); - memset(m->defaults, 0, size); - m->defaults_pyobjects = pyobjects; - return m->defaults; -} -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->defaults_tuple = tuple; - Py_INCREF(tuple); -} -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->defaults_kwdict = dict; - Py_INCREF(dict); -} -static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->func_annotations = dict; - Py_INCREF(dict); -} - -/* PyObject_GenericGetAttrNoDict */ -#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 -static PyObject *__Pyx_RaiseGenericGetAttributeError(PyTypeObject *tp, PyObject *attr_name) { - PyErr_Format(PyExc_AttributeError, -#if PY_MAJOR_VERSION >= 3 - "'%.50s' object has no attribute '%U'", - tp->tp_name, attr_name); + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); #else - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, PyString_AS_STRING(attr_name)); + PyErr_SetExcInfo(local_type, local_value, local_tb); #endif - return NULL; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name) { - PyObject *descr; - PyTypeObject *tp = Py_TYPE(obj); - if (unlikely(!PyString_Check(attr_name))) { - return PyObject_GenericGetAttr(obj, attr_name); - } - assert(!tp->tp_dictoffset); - descr = _PyType_Lookup(tp, attr_name); - if (unlikely(!descr)) { - return __Pyx_RaiseGenericGetAttributeError(tp, attr_name); - } - Py_INCREF(descr); - #if PY_MAJOR_VERSION < 3 - if (likely(PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_HAVE_CLASS))) - #endif - { - descrgetfunc f = Py_TYPE(descr)->tp_descr_get; - if (unlikely(f)) { - PyObject *res = f(descr, obj, (PyObject *)tp); - Py_DECREF(descr); - return res; - } - } - return descr; + return 0; +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; } -#endif /* TypeImport */ #ifndef __PYX_HAVE_RT_ImportType @@ -8249,24 +7343,24 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value) { + const uint32_t neg_one = (uint32_t) ((uint32_t) 0 - (uint32_t) 1), const_zero = (uint32_t) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { - if (sizeof(long) < sizeof(long)) { + if (sizeof(uint32_t) < sizeof(long)) { return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(unsigned long)) { + } else if (sizeof(uint32_t) <= sizeof(unsigned long)) { return PyLong_FromUnsignedLong((unsigned long) value); #ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + } else if (sizeof(uint32_t) <= sizeof(unsigned PY_LONG_LONG)) { return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); #endif } } else { - if (sizeof(long) <= sizeof(long)) { + if (sizeof(uint32_t) <= sizeof(long)) { return PyInt_FromLong((long) value); #ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + } else if (sizeof(uint32_t) <= sizeof(PY_LONG_LONG)) { return PyLong_FromLongLong((PY_LONG_LONG) value); #endif } @@ -8274,7 +7368,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { { int one = 1; int little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(long), + return _PyLong_FromByteArray(bytes, sizeof(uint32_t), little, !is_unsigned); } } @@ -8643,59 +7737,437 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { #endif } } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(int), - little, !is_unsigned); - } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); + } +} + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__NPY_TYPES(enum NPY_TYPES value) { + const enum NPY_TYPES neg_one = (enum NPY_TYPES) ((enum NPY_TYPES) 0 - (enum NPY_TYPES) 1), const_zero = (enum NPY_TYPES) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(enum NPY_TYPES) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(enum NPY_TYPES) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(enum NPY_TYPES) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(enum NPY_TYPES) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(enum NPY_TYPES) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(enum NPY_TYPES), + little, !is_unsigned); + } +} + +/* CIntFromPy */ +static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *x) { + const unsigned int neg_one = (unsigned int) ((unsigned int) 0 - (unsigned int) 1), const_zero = (unsigned int) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(unsigned int) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(unsigned int, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (unsigned int) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (unsigned int) 0; + case 1: __PYX_VERIFY_RETURN_INT(unsigned int, digit, digits[0]) + case 2: + if (8 * sizeof(unsigned int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned int) >= 2 * PyLong_SHIFT) { + return (unsigned int) (((((unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(unsigned int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned int) >= 3 * PyLong_SHIFT) { + return (unsigned int) (((((((unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(unsigned int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned int) >= 4 * PyLong_SHIFT) { + return (unsigned int) (((((((((unsigned int)digits[3]) << PyLong_SHIFT) | (unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (unsigned int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(unsigned int) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned int) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (unsigned int) 0; + case -1: __PYX_VERIFY_RETURN_INT(unsigned int, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(unsigned int, digit, +digits[0]) + case -2: + if (8 * sizeof(unsigned int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned int) - 1 > 2 * PyLong_SHIFT) { + return (unsigned int) (((unsigned int)-1)*(((((unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(unsigned int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned int) - 1 > 2 * PyLong_SHIFT) { + return (unsigned int) ((((((unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(unsigned int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned int) - 1 > 3 * PyLong_SHIFT) { + return (unsigned int) (((unsigned int)-1)*(((((((unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(unsigned int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned int) - 1 > 3 * PyLong_SHIFT) { + return (unsigned int) ((((((((unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(unsigned int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned int) - 1 > 4 * PyLong_SHIFT) { + return (unsigned int) (((unsigned int)-1)*(((((((((unsigned int)digits[3]) << PyLong_SHIFT) | (unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(unsigned int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(unsigned int) - 1 > 4 * PyLong_SHIFT) { + return (unsigned int) ((((((((((unsigned int)digits[3]) << PyLong_SHIFT) | (unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + } +#endif + if (sizeof(unsigned int) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned int) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + unsigned int val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (unsigned int) -1; + } + } else { + unsigned int val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (unsigned int) -1; + val = __Pyx_PyInt_As_unsigned_int(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned int"); + return (unsigned int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned int"); + return (unsigned int) -1; } -/* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__NPY_TYPES(enum NPY_TYPES value) { - const enum NPY_TYPES neg_one = (enum NPY_TYPES) ((enum NPY_TYPES) 0 - (enum NPY_TYPES) 1), const_zero = (enum NPY_TYPES) 0; +/* CIntFromPy */ +static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *x) { + const char neg_one = (char) ((char) 0 - (char) 1), const_zero = (char) 0; const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(enum NPY_TYPES) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(enum NPY_TYPES) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(char) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(char, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (char) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (char) 0; + case 1: __PYX_VERIFY_RETURN_INT(char, digit, digits[0]) + case 2: + if (8 * sizeof(char) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) >= 2 * PyLong_SHIFT) { + return (char) (((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(char) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) >= 3 * PyLong_SHIFT) { + return (char) (((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(char) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) >= 4 * PyLong_SHIFT) { + return (char) (((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (char) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(char) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(enum NPY_TYPES) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + } else if (sizeof(char) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) #endif - } - } else { - if (sizeof(enum NPY_TYPES) <= sizeof(long)) { - return PyInt_FromLong((long) value); + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (char) 0; + case -1: __PYX_VERIFY_RETURN_INT(char, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(char, digit, +digits[0]) + case -2: + if (8 * sizeof(char) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 2 * PyLong_SHIFT) { + return (char) (((char)-1)*(((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(char) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 2 * PyLong_SHIFT) { + return (char) ((((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(char) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { + return (char) (((char)-1)*(((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(char) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { + return (char) ((((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 4 * PyLong_SHIFT) { + return (char) (((char)-1)*(((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(char) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 4 * PyLong_SHIFT) { + return (char) ((((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + } +#endif + if (sizeof(char) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(char, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(enum NPY_TYPES) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); + } else if (sizeof(char) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(char, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif + } } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + char val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (char) -1; + } + } else { + char val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (char) -1; + val = __Pyx_PyInt_As_char(tmp); + Py_DECREF(tmp); + return val; } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(enum NPY_TYPES), - little, !is_unsigned); - } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to char"); + return (char) -1; } /* CIntFromPy */ -static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *x) { - const unsigned int neg_one = (unsigned int) ((unsigned int) 0 - (unsigned int) 1), const_zero = (unsigned int) 0; +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { + const size_t neg_one = (size_t) ((size_t) 0 - (size_t) 1), const_zero = (size_t) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { - if (sizeof(unsigned int) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(unsigned int, long, PyInt_AS_LONG(x)) + if (sizeof(size_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) } else { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { goto raise_neg_overflow; } - return (unsigned int) val; + return (size_t) val; } } else #endif @@ -8704,32 +8176,32 @@ static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *x) { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { - case 0: return (unsigned int) 0; - case 1: __PYX_VERIFY_RETURN_INT(unsigned int, digit, digits[0]) + case 0: return (size_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) case 2: - if (8 * sizeof(unsigned int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned int) >= 2 * PyLong_SHIFT) { - return (unsigned int) (((((unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0])); + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); } } break; case 3: - if (8 * sizeof(unsigned int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned int) >= 3 * PyLong_SHIFT) { - return (unsigned int) (((((((unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0])); + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); } } break; case 4: - if (8 * sizeof(unsigned int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned int) >= 4 * PyLong_SHIFT) { - return (unsigned int) (((((((((unsigned int)digits[3]) << PyLong_SHIFT) | (unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0])); + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); } } break; @@ -8743,86 +8215,86 @@ static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *x) { { int result = PyObject_RichCompareBool(x, Py_False, Py_LT); if (unlikely(result < 0)) - return (unsigned int) -1; + return (size_t) -1; if (unlikely(result == 1)) goto raise_neg_overflow; } #endif - if (sizeof(unsigned int) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(unsigned int, unsigned long, PyLong_AsUnsignedLong(x)) + if (sizeof(size_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned int) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(unsigned int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) #endif } } else { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { - case 0: return (unsigned int) 0; - case -1: __PYX_VERIFY_RETURN_INT(unsigned int, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(unsigned int, digit, +digits[0]) + case 0: return (size_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) case -2: - if (8 * sizeof(unsigned int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned int) - 1 > 2 * PyLong_SHIFT) { - return (unsigned int) (((unsigned int)-1)*(((((unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); } } break; case 2: - if (8 * sizeof(unsigned int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned int) - 1 > 2 * PyLong_SHIFT) { - return (unsigned int) ((((((unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); } } break; case -3: - if (8 * sizeof(unsigned int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned int) - 1 > 3 * PyLong_SHIFT) { - return (unsigned int) (((unsigned int)-1)*(((((((unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); } } break; case 3: - if (8 * sizeof(unsigned int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned int) - 1 > 3 * PyLong_SHIFT) { - return (unsigned int) ((((((((unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); } } break; case -4: - if (8 * sizeof(unsigned int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned int) - 1 > 4 * PyLong_SHIFT) { - return (unsigned int) (((unsigned int)-1)*(((((((((unsigned int)digits[3]) << PyLong_SHIFT) | (unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); } } break; case 4: - if (8 * sizeof(unsigned int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned int) - 1 > 4 * PyLong_SHIFT) { - return (unsigned int) ((((((((((unsigned int)digits[3]) << PyLong_SHIFT) | (unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); } } break; } #endif - if (sizeof(unsigned int) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(unsigned int, long, PyLong_AsLong(x)) + if (sizeof(size_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned int) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(unsigned int, PY_LONG_LONG, PyLong_AsLongLong(x)) + } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif } } @@ -8831,7 +8303,7 @@ static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *x) { PyErr_SetString(PyExc_RuntimeError, "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); #else - unsigned int val; + size_t val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); #if PY_MAJOR_VERSION < 3 if (likely(v) && !PyLong_Check(v)) { @@ -8851,24 +8323,24 @@ static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *x) { return val; } #endif - return (unsigned int) -1; + return (size_t) -1; } } else { - unsigned int val; + size_t val; PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (unsigned int) -1; - val = __Pyx_PyInt_As_unsigned_int(tmp); + if (!tmp) return (size_t) -1; + val = __Pyx_PyInt_As_size_t(tmp); Py_DECREF(tmp); return val; } raise_overflow: PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned int"); - return (unsigned int) -1; + "value too large to convert to size_t"); + return (size_t) -1; raise_neg_overflow: PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned int"); - return (unsigned int) -1; + "can't convert negative value to size_t"); + return (size_t) -1; } /* CIntFromPy */ @@ -9060,6 +8532,37 @@ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { return (int) -1; } +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } +} + /* CIntFromPy */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index f27d7e400f..cc4ba9bbb4 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -7,29 +7,27 @@ """General functions used for any2vec models.""" +# +# This is here to support older versions of the MSVC compiler that don't have stdint.h. +# +cdef extern from "stdint_wrapper.h": + ctypedef unsigned int uint32_t + ctypedef signed char int8_t + from six import PY2 import numpy as np cimport numpy as np -cdef _byte_to_int_py3(b): - return b - -cdef _byte_to_int_py2(b): - return ord(b) - -_byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 - - -cpdef ft_hash(unicode string): - """Calculate hash based on `string`. +cpdef ft_hash_bytes(bytes bytez): + """Calculate hash based on `bytez`. Reproduce `hash method from Facebook fastText implementation `_. Parameters ---------- - string : unicode - The string whose hash needs to be calculated. + bytez : bytes + The string whose hash needs to be calculated, encoded as UTF-8. Returns ------- @@ -37,10 +35,12 @@ cpdef ft_hash(unicode string): The hash of the string. """ - cdef unsigned int h = 2166136261 - for c in string.encode("utf-8"): - h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) - h = np.uint32(h * np.uint32(16777619)) + cdef uint32_t h = 2166136261 + cdef char b + + for b in bytez: + h = h ^ (b) + h = h * 16777619 return h @@ -92,3 +92,56 @@ cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): for i in range(0, len(extended_word) - ngram_length + 1): ngrams.append(extended_word[i:i + ngram_length]) return ngrams + +# +# UTF-8 bytes that begin with 10 are subsequent bytes of a multi-byte sequence, +# as opposed to a new character. +# +cdef unsigned char _MB_MASK = 0xC0 +cdef unsigned char _MB_START = 0x80 + + +cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): + """Computes ngrams for a word. + + Ported from the original FB implementation. + + Parameters + ---------- + word : str + A unicode string. + min_n : unsigned int + The minimum ngram length. + max_n : unsigned int + The maximum ngram length. + + Returns: + -------- + list of str + A list of ngrams, where each ngram is a list of **bytes**. + + See Also + -------- + `Original implementation `__ + + """ + cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") + cdef const unsigned char *bytez = utf8_word + cdef size_t num_bytes = len(utf8_word) + cdef size_t j, i, n + + ngrams = [] + for i in range(num_bytes): + if bytez[i] & _MB_MASK == _MB_START: + continue + + j, n = i, 1 + while j < num_bytes and n <= max_n: + j += 1 + while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: + j += 1 + if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + ngram = bytes(bytez[i:j]) + ngrams.append(ngram) + n += 1 + return ngrams diff --git a/gensim/models/fasttext.py b/gensim/models/fasttext.py index 1b03268442..807ebec32d 100644 --- a/gensim/models/fasttext.py +++ b/gensim/models/fasttext.py @@ -98,7 +98,7 @@ from gensim.models.word2vec import Word2VecVocab, Word2VecTrainables, train_sg_pair, train_cbow_pair from gensim.models.keyedvectors import FastTextKeyedVectors from gensim.models.base_any2vec import BaseWordEmbeddingsModel -from gensim.models.utils_any2vec import _compute_ngrams, _ft_hash, _ft_hash_broken +from gensim.models.utils_any2vec import ft_ngram_hashes from smart_open import smart_open from gensim.utils import deprecated, call_on_class_only @@ -554,8 +554,6 @@ def _clear_post_train(self): self.wv.buckets_word = None def estimate_memory(self, vocab_size=None, report=None): - hash_fn = _ft_hash if self.wv.compatible_hash else _ft_hash_broken - vocab_size = vocab_size or len(self.wv.vocab) vec_size = self.vector_size * np.dtype(np.float32).itemsize l1_size = self.trainables.layer1_size * np.dtype(np.float32).itemsize @@ -574,9 +572,15 @@ def estimate_memory(self, vocab_size=None, report=None): buckets = set() num_ngrams = 0 for word in self.wv.vocab: - ngrams = _compute_ngrams(word, self.wv.min_n, self.wv.max_n) - num_ngrams += len(ngrams) - buckets.update(hash_fn(ng) % self.trainables.bucket for ng in ngrams) + hashes = ft_ngram_hashes( + word, + self.wv.min_n, + self.wv.max_n, + self.trainables.bucket, + self.wv.compatible_hash + ) + num_ngrams += len(hashes) + buckets.update(hashes) num_buckets = len(buckets) report['syn0_ngrams'] = num_buckets * vec_size # A tuple (48 bytes) with num_ngrams_word ints (8 bytes) for each word @@ -704,6 +708,14 @@ def train(self, sentences=None, corpus_file=None, total_examples=None, total_wor >>> model.train(sentences, total_examples=model.corpus_count, epochs=model.epochs) """ + cant_train = hasattr(self.trainables, 'syn1neg') and self.trainables.syn1neg is None + if cant_train: + raise ValueError( + 'this model cannot be trained any further, ' + 'if this is a native model, try loading it with ' + 'FastText.load_fasttext_format(path, full_model=True)' + ) + super(FastText, self).train( sentences=sentences, corpus_file=corpus_file, total_examples=total_examples, total_words=total_words, epochs=epochs, start_alpha=start_alpha, end_alpha=end_alpha, word_count=word_count, @@ -754,7 +766,7 @@ def __contains__(self, word): return self.wv.__contains__(word) @classmethod - def load_fasttext_format(cls, model_file, encoding='utf8'): + def load_fasttext_format(cls, model_file, encoding='utf8', full_model=True): """Load the input-hidden weight matrix from Facebook's native fasttext `.bin` and `.vec` output files. Notes @@ -770,6 +782,9 @@ def load_fasttext_format(cls, model_file, encoding='utf8'): as Gensim requires only `.bin` file to the load entire fastText model. encoding : str, optional Specifies the file encoding. + full_model : boolean, optional + If False, skips loading the hidden output matrix. This saves a fair bit + of CPU time and RAM, but prevents training continuation. Returns ------- @@ -777,7 +792,7 @@ def load_fasttext_format(cls, model_file, encoding='utf8'): The loaded model. """ - return _load_fasttext_format(model_file, encoding=encoding) + return _load_fasttext_format(model_file, encoding=encoding, full_model=full_model) def load_binary_data(self, encoding='utf8'): """Load data from a binary file created by Facebook's native FastText. @@ -959,7 +974,7 @@ def _pad_ones(m, new_shape): return vstack([m, suffix]) -def _load_fasttext_format(model_file, encoding='utf-8'): +def _load_fasttext_format(model_file, encoding='utf-8', full_model=True): """Load the input-hidden weight matrix from Facebook's native fasttext `.bin` and `.vec` output files. Parameters @@ -971,16 +986,20 @@ def _load_fasttext_format(model_file, encoding='utf-8'): as Gensim requires only `.bin` file to the load entire fastText model. encoding : str, optional Specifies the file encoding. + full_model : boolean, optional + If False, skips loading the hidden output matrix. This saves a fair bit + of CPU time and RAM, but prevents training continuation. Returns ------- :class: `~gensim.models.fasttext.FastText` The loaded model. + """ if not model_file.endswith('.bin'): model_file += '.bin' with smart_open(model_file, 'rb') as fin: - m = gensim.models._fasttext_bin.load(fin, encoding=encoding) + m = gensim.models._fasttext_bin.load(fin, encoding=encoding, full_model=full_model) model = FastText( size=m.dim, diff --git a/gensim/models/keyedvectors.py b/gensim/models/keyedvectors.py index 65711baec6..ff79212cea 100644 --- a/gensim/models/keyedvectors.py +++ b/gensim/models/keyedvectors.py @@ -182,9 +182,7 @@ from gensim.models.utils_any2vec import ( _save_word2vec_format, _load_word2vec_format, - _compute_ngrams, - _ft_hash, - _ft_hash_broken + ft_ngram_hashes, ) from gensim.similarities.termsim import TermSimilarityIndex, SparseTermSimilarityMatrix @@ -2029,9 +2027,8 @@ def __contains__(self, word): if word in self.vocab: return True else: - hash_fn = _ft_hash if self.compatible_hash else _ft_hash_broken - char_ngrams = _compute_ngrams(word, self.min_n, self.max_n) - return any(hash_fn(ng) % self.bucket in self.hash2index for ng in char_ngrams) + hashes = ft_ngram_hashes(word, self.min_n, self.max_n, self.bucket, self.compatible_hash) + return any(h in self.hash2index for h in hashes) def save(self, *args, **kwargs): """Save object. @@ -2073,23 +2070,18 @@ def word_vec(self, word, use_norm=False): If word and all ngrams not in vocabulary. """ - hash_fn = _ft_hash if self.compatible_hash else _ft_hash_broken - if word in self.vocab: return super(FastTextKeyedVectors, self).word_vec(word, use_norm) elif self.bucket == 0: raise KeyError('cannot calculate vector for OOV word without ngrams') else: - # from gensim.models.fasttext import compute_ngrams word_vec = np.zeros(self.vectors_ngrams.shape[1], dtype=np.float32) - ngrams = _compute_ngrams(word, self.min_n, self.max_n) if use_norm: ngram_weights = self.vectors_ngrams_norm else: ngram_weights = self.vectors_ngrams ngrams_found = 0 - for ngram in ngrams: - ngram_hash = hash_fn(ngram) % self.bucket + for ngram_hash in ft_ngram_hashes(word, self.min_n, self.max_n, self.bucket, self.compatible_hash): if ngram_hash in self.hash2index: word_vec += ngram_weights[self.hash2index[ngram_hash]] ngrams_found += 1 @@ -2147,7 +2139,7 @@ def init_ngrams_weights(self, seed): self.min_n, self.max_n, self.bucket, - _ft_hash if self.compatible_hash else _ft_hash_broken, + self.compatible_hash, self.hash2index ) self.num_ngram_vectors = len(ngram_indices) @@ -2170,7 +2162,7 @@ def update_ngrams_weights(self, seed, old_vocab_len): self.min_n, self.max_n, self.bucket, - _ft_hash if self.compatible_hash else _ft_hash_broken, + self.compatible_hash, self.hash2index ) num_new_ngrams = len(new_ngram_hashes) @@ -2232,7 +2224,7 @@ def init_post_load(self, vectors, match_gensim=False): self.min_n, self.max_n, self.bucket, - _ft_hash if self.compatible_hash else _ft_hash_broken, + self.compatible_hash, dict(), # we don't care what goes here in this case ) ngram_hashes = sorted(set(ngram_hashes)) @@ -2254,19 +2246,16 @@ def adjust_vectors(self): if self.bucket == 0: return - hash_fn = _ft_hash if self.compatible_hash else _ft_hash_broken - for w, v in self.vocab.items(): word_vec = np.copy(self.vectors_vocab[v.index]) - ngrams = _compute_ngrams(w, self.min_n, self.max_n) - for ngram in ngrams: - ngram_index = self.hash2index[hash_fn(ngram) % self.bucket] - word_vec += self.vectors_ngrams[ngram_index] - word_vec /= len(ngrams) + 1 + ngram_hashes = ft_ngram_hashes(w, self.min_n, self.max_n, self.bucket, self.compatible_hash) + for nh in ngram_hashes: + word_vec += self.vectors_ngrams[self.hash2index[nh]] + word_vec /= len(ngram_hashes) + 1 self.vectors[v.index] = word_vec -def _process_fasttext_vocab(iterable, min_n, max_n, num_buckets, hash_fn, hash2index): +def _process_fasttext_vocab(iterable, min_n, max_n, num_buckets, compatible_hash, hash2index): """ Performs a common operation for FastText weight initialization and updates: scan the vocabulary, calculate ngrams and their hashes, keep @@ -2283,8 +2272,9 @@ def _process_fasttext_vocab(iterable, min_n, max_n, num_buckets, hash_fn, hash2i The maximum length of ngrams. num_buckets : int The number of buckets used by the model. - hash_fn : callable - Used to hash ngrams to buckets. + compatible_hash : boolean + True for compatibility with the Facebook implementation. + False for compatibility with the old Gensim implementation. hash2index : dict Updated in-place. @@ -2310,8 +2300,7 @@ def _process_fasttext_vocab(iterable, min_n, max_n, num_buckets, hash_fn, hash2i for word, vocab in iterable: wi = [] - for ngram in _compute_ngrams(word, min_n, max_n): - ngram_hash = hash_fn(ngram) % num_buckets + for ngram_hash in ft_ngram_hashes(word, min_n, max_n, num_buckets, compatible_hash): if ngram_hash not in hash2index: # # This is a new ngram. Reserve a new index in hash2index. diff --git a/gensim/models/stdint_wrapper.h b/gensim/models/stdint_wrapper.h new file mode 100644 index 0000000000..0b9832dedf --- /dev/null +++ b/gensim/models/stdint_wrapper.h @@ -0,0 +1,19 @@ +/* + * This file is here to support older versions of the MSVC compiler that don't + * have stdint.h. + */ +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned int uint32_t; + typedef char int8_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int32 uint32_t; + typedef char int8_t; + #endif + #endif +#else + #include +#endif diff --git a/gensim/models/utils_any2vec.py b/gensim/models/utils_any2vec.py index 74d0effcff..88242c31ea 100644 --- a/gensim/models/utils_any2vec.py +++ b/gensim/models/utils_any2vec.py @@ -4,7 +4,26 @@ # Author: Shiva Manne # Copyright (C) 2018 RaRe Technologies s.r.o. -"""General functions used for any2vec models.""" +"""General functions used for any2vec models. + +One of the goals of this module is to provide an abstraction over the Cython +extensions for FastText. If they are not available, then the module substitutes +slower Python versions in their place. + +Another related set of FastText functionality is computing ngrams for a word. +The :py:func:`compute_ngrams` and :py:func:`compute_ngrams_bytes` hashes achieve that. + +Closely related is the functionality for hashing ngrams, implemented by the +:py:func:`ft_hash` and :py:func:`ft_hash_broken` functions. +The module exposes "working" and "broken" hash functions in order to maintain +backwards compatibility with older versions of Gensim. + +For compatibility with older Gensim, use :py:func:`compute_ngrams` and +:py:func:`ft_hash_broken` to has each ngram. For compatibility with the +current Facebook implementation, use :py:func:`compute_ngrams_bytes` and +:py:func:`ft_hash_bytes`. + +""" import logging import numpy as np @@ -18,6 +37,14 @@ logger = logging.getLogger(__name__) +# +# UTF-8 bytes that begin with 10 are subsequent bytes of a multi-byte sequence, +# as opposed to a new character. +# +_MB_MASK = 0xC0 +_MB_START = 0x80 + + def _byte_to_int_py3(b): return b @@ -29,19 +56,23 @@ def _byte_to_int_py2(b): _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 +def _is_utf8_continue(b): + return _byte_to_int(b) & _MB_MASK == _MB_START + + # -# Define this here so we can unittest here. Only use this function if the -# faster C version fails to import. +# Define this here so we can unittest this function directly. +# Only use this function if the faster C version fails to import. # -def _ft_hash_py(string): - """Calculate hash based on `string`. +def _ft_hash_bytes_py(bytez): + """Calculate hash based on `bytez`. Reproduce `hash method from Facebook fastText implementation `_. Parameters ---------- - string : str - The string whose hash needs to be calculated. + bytez : bytes + The string whose hash needs to be calculated, encoded as UTF-8. Returns ------- @@ -51,14 +82,14 @@ def _ft_hash_py(string): """ old_settings = np.seterr(all='ignore') h = np.uint32(2166136261) - for c in string.encode('utf-8'): - h = h ^ np.uint32(np.int8(_byte_to_int(c))) + for b in bytez: + h = h ^ np.uint32(np.int8(_byte_to_int(b))) h = h * np.uint32(16777619) np.seterr(**old_settings) return h -def _ft_hash_py_broken(string): +def _ft_hash_broken_py(string): """Calculate hash based on `string`. Reproduce `hash method from Facebook fastText implementation `_. @@ -86,46 +117,133 @@ def _ft_hash_py_broken(string): return h +def _compute_ngrams_py(word, min_n, max_n): + """Get the list of all possible ngrams for a given word. + Parameters + ---------- + word : str + The word whose ngrams need to be computed. + min_n : int + Minimum character length of the ngrams. + max_n : int + Maximum character length of the ngrams. + Returns + ------- + list of str + Sequence of character ngrams. + """ + BOW, EOW = ('<', '>') # Used by FastText to attach to all words as prefix and suffix + extended_word = BOW + word + EOW + ngrams = [] + for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): + for i in range(0, len(extended_word) - ngram_length + 1): + ngrams.append(extended_word[i:i + ngram_length]) + return ngrams + + +def _compute_ngrams_bytes_py(word, min_n, max_n): + """Computes ngrams for a word. + + Ported from the original FB implementation. + + Parameters + ---------- + word : str + A unicode string. + min_n : unsigned int + The minimum ngram length. + max_n : unsigned int + The maximum ngram length. + + Returns: + -------- + list of str + A list of ngrams, where each ngram is a list of **bytes**. + + See Also + -------- + `Original implementation `__ # noqa: E501 + + """ + utf8_word = ('<%s>' % word).encode("utf-8") + num_bytes = len(utf8_word) + n = 0 + + ngrams = [] + for i in range(num_bytes): + if _is_utf8_continue(utf8_word[i]): + continue + + j, n = i, 1 + while j < num_bytes and n <= max_n: + j += 1 + while j < num_bytes and _is_utf8_continue(utf8_word[j]): + j += 1 + if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + ngram = bytes(utf8_word[i:j]) + ngrams.append(ngram) + n += 1 + return ngrams + + +# +# Internally, we use the following convention to abstract away the presence +# or absence of the Cython extensions: +# +# - _function_cy: Imported from Cython extension +# - _function_py: Implemented in Python +# - function: Exported by this module. +# try: from gensim.models._utils_any2vec import ( - ft_hash as _ft_hash_cy, - ft_hash_broken as _ft_hash_cy_broken, - compute_ngrams as _compute_ngrams + compute_ngrams as _compute_ngrams_cy, + compute_ngrams_bytes as _compute_ngrams_bytes_cy, + ft_hash_broken as _ft_hash_broken_cy, + ft_hash_bytes as _ft_hash_bytes_cy, ) - _ft_hash = _ft_hash_cy - _ft_hash_broken = _ft_hash_cy_broken + ft_hash_bytes = _ft_hash_bytes_cy + ft_hash_broken = _ft_hash_broken_cy + compute_ngrams = _compute_ngrams_cy + compute_ngrams_bytes = _compute_ngrams_bytes_cy + FAST_VERSION = 0 except ImportError: + # failed... fall back to plain python FAST_VERSION = -1 + ft_hash_bytes = _ft_hash_bytes_py + ft_hash_broken = _ft_hash_broken_py + compute_ngrams = _compute_ngrams_py + compute_ngrams_bytes = _compute_ngrams_bytes_py - _ft_hash = _ft_hash_py - _ft_hash_broken = _ft_hash_py_broken - # failed... fall back to plain python - def _compute_ngrams(word, min_n, max_n): - """Get the list of all possible ngrams for a given word. +def ft_ngram_hashes(word, minn, maxn, num_buckets, fb_compatible=True): + """Calculate the ngrams of the word and hash them. - Parameters - ---------- - word : str - The word whose ngrams need to be computed. - min_n : int - Minimum character length of the ngrams. - max_n : int - Maximum character length of the ngrams. - - Returns - ------- - list of str - Sequence of character ngrams. + Parameters + ---------- + word : str + The word to calculate ngram hashes for. + minn : int + Minimum ngram length + maxn : int + Maximum ngram length + num_buckets : int + The number of buckets + fb_compatible : boolean, optional + True for compatibility with the Facebook implementation. + False for compatibility with the old Gensim implementation. - """ - BOW, EOW = ('<', '>') # Used by FastText to attach to all words as prefix and suffix - extended_word = BOW + word + EOW - ngrams = [] - for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): - for i in range(0, len(extended_word) - ngram_length + 1): - ngrams.append(extended_word[i:i + ngram_length]) - return ngrams + Returns + ------- + A list of hashes (integers), one per each detected ngram. + + """ + if fb_compatible: + encoded_ngrams = compute_ngrams_bytes(word, minn, maxn) + hashes = [ft_hash_bytes(n) % num_buckets for n in encoded_ngrams] + else: + text_ngrams = compute_ngrams(word, minn, maxn) + hashes = [ft_hash_broken(n) % num_buckets for n in text_ngrams] + return hashes def _save_word2vec_format(fname, vocab, vectors, fvocab=None, binary=False, total_vec=None): diff --git a/gensim/test/test_data/fb-ngrams.txt b/gensim/test/test_data/fb-ngrams.txt new file mode 100644 index 0000000000..3296d85c6a --- /dev/null +++ b/gensim/test/test_data/fb-ngrams.txt @@ -0,0 +1,89 @@ + +test +test 0 0 0 0 0 + -0.058167 0.084801 -0.10452 0.085963 -0.14475 +tes 0.066033 0.011132 0.13224 0.19839 -0.037827 +test -0.12628 0.053912 0.051677 0.24844 0.083743 +test> -0.016497 0.1322 0.21391 0.20869 -0.044566 +est -0.12915 -0.02457 0.10145 0.23414 0.17121 +est> -0.1776 0.19906 -0.15572 0.16449 -0.12887 +st> -0.13712 0.028013 -0.041927 0.11244 0.12261 + + +at the +at the 0 0 0 0 0 + 0.17667 0.026172 -0.15457 0.081664 -0.1315 + th 0.1095 0.077397 0.052745 -0.043956 0.033436 + the -0.15504 0.11166 -0.16434 0.089162 -0.16199 + the> 0.097803 -0.054657 -0.028828 0.21607 0.11286 +the -0.046122 -0.071463 0.20759 0.25732 0.23174 +the> 0.04535 0.15192 -0.0021161 0.014343 0.12982 +he> -0.046122 -0.071463 0.20759 0.25732 0.23174 + + +atnthe +atnthe 0 0 0 0 0 + -0.065392 -0.085291 0.088378 0.024628 0.12715 +nth 0.067156 -0.068108 0.16394 0.18192 0.18902 +nthe 0.13526 -0.023216 -0.10788 0.24512 -0.022174 +nthe> -0.20296 -0.11805 0.20277 0.063027 -0.041727 +the -0.046122 -0.071463 0.20759 0.25732 0.23174 +the> 0.04535 0.15192 -0.0021161 0.014343 0.12982 +he> -0.046122 -0.071463 0.20759 0.25732 0.23174 + + +тест +тест 0 0 0 0 0 +<те -0.17079 -0.029842 0.10298 -0.12632 -0.14488 +<тес -0.026151 0.13071 0.20833 0.17143 0.040533 +<тест 0.17667 0.026172 -0.15457 0.081664 -0.1315 +<тест> -0.065392 -0.085291 0.088378 0.024628 0.12715 +тес -0.08172 -0.13699 0.14081 -0.036949 -0.037403 +тест -0.11209 0.068818 -0.137 -0.088165 0.13144 +тест> -0.20296 -0.11805 0.20277 0.063027 -0.041727 +ест -0.17079 -0.029842 0.10298 -0.12632 -0.14488 +ест> 0.16922 -0.1162 0.1462 0.1085 0.02952 +ст> -0.17261 0.086891 0.21772 0.10036 0.19114 + + +テスト +テスト 0 0 0 0 0 +<テス 0.066033 0.011132 0.13224 0.19839 -0.037827 +<テスト -0.12915 -0.02457 0.10145 0.23414 0.17121 +<テスト> -0.10241 0.0090735 0.097437 0.055768 0.03622 +テスト 0.048443 0.15183 -0.058779 -0.1095 0.10566 +テスト> -0.12628 0.053912 0.051677 0.24844 0.083743 +スト> 0.12972 0.18282 -0.18266 -0.072433 0.19318 + + +試し +試し 0 0 0 0 0 +<試し 0.1095 0.077397 0.052745 -0.043956 0.033436 +<試し> 0.11952 0.15093 -0.075759 0.18327 0.12471 +試し> -0.14656 0.072651 -0.16777 0.20026 0.21199 + diff --git a/gensim/test/test_fasttext.py b/gensim/test/test_fasttext.py index 67b035549b..5437a9b3c8 100644 --- a/gensim/test/test_fasttext.py +++ b/gensim/test/test_fasttext.py @@ -59,6 +59,21 @@ def setUp(self): self.test_model = FT_gensim.load_fasttext_format(self.test_model_file) self.test_new_model_file = datapath('lee_fasttext_new') + def test_native_partial_model(self): + """Can we skip loading the NN and still get a working model?""" + model = FT_gensim.load_fasttext_format(self.test_model_file, full_model=False) + + # + # Training continuation should be impossible + # + self.assertIsNone(model.trainables.syn1neg) + self.assertRaises(ValueError, model.train, sentences, + total_examples=model.corpus_count, epochs=model.epochs) + + model.wv['green'] + model.wv['foobar'] + model.wv['thisworddoesnotexist'] + def test_training(self): model = FT_gensim(size=10, min_count=1, hs=1, negative=0, seed=42, workers=1) model.build_vocab(sentences) diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index 74ca8b941f..c23087580f 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -6,8 +6,9 @@ """ Automated tests for checking various utils functions. """ +from __future__ import unicode_literals - +import sys import logging import unittest @@ -19,6 +20,8 @@ import gensim.models.utils_any2vec +import smart_open + DISABLE_CYTHON_TESTS = getattr(gensim.models.utils_any2vec, 'FAST_VERSION', None) == -1 @@ -269,12 +272,13 @@ def hash_main(alg): import six assert six.PY3, 'this only works under Py3' + assert not DISABLE_CYTHON_TESTS, 'this only works if Cython extensions available' hashmap = { - 'py': gensim.models.utils_any2vec._ft_hash_py, - 'py_broken': gensim.models.utils_any2vec._ft_hash_py_broken, - 'cy': gensim.models.utils_any2vec._ft_hash_py, - 'cy_broken': gensim.models.utils_any2vec._ft_hash_py_broken, + 'py_broken': gensim.models.utils_any2vec._ft_hash_broken_py, + 'py_bytes': gensim.models.utils_any2vec._ft_hash_bytes_py, + 'cy_broken': gensim.models.utils_any2vec._ft_hash_broken_py, + 'cy_bytes': gensim.models.utils_any2vec._ft_hash_bytes_cy, } try: fun = hashmap[alg] @@ -282,7 +286,11 @@ def hash_main(alg): raise KeyError('invalid alg: %r expected one of %r' % (alg, sorted(hashmap))) for line in sys.stdin: - for word in line.rstrip().split(' '): + if 'bytes' in alg: + words = line.encode('utf-8').rstrip().split(b' ') + else: + words = line.rstrip().split(' ') + for word in words: print('u%r: %r,' % (word, fun(word))) @@ -293,7 +301,7 @@ def setUp(self): # # $ echo word1 ... wordN | python -c 'from gensim.test.test_utils import hash_main;hash_main("alg")' # noqa: E501 # - # where alg is one of py, py_broken, cy, cy_broken. + # where alg is one of py_bytes, py_broken, cy_bytes, cy_broken. # self.expected = { @@ -330,24 +338,241 @@ def setUp(self): } def test_python(self): - actual = {k: gensim.models.utils_any2vec._ft_hash_py(k) for k in self.expected} + actual = {k: gensim.models.utils_any2vec._ft_hash_bytes_py(k.encode('utf-8')) for k in self.expected} self.assertEqual(self.expected, actual) @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') def test_cython(self): - actual = {k: gensim.models.utils_any2vec._ft_hash_cy(k) for k in self.expected} + actual = {k: gensim.models.utils_any2vec._ft_hash_bytes_cy(k.encode('utf-8')) for k in self.expected} self.assertEqual(self.expected, actual) def test_python_broken(self): - actual = {k: gensim.models.utils_any2vec._ft_hash_py_broken(k) for k in self.expected} + actual = {k: gensim.models.utils_any2vec._ft_hash_broken_py(k) for k in self.expected} self.assertEqual(self.expected_broken, actual) @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') def test_cython_broken(self): - actual = {k: gensim.models.utils_any2vec._ft_hash_cy_broken(k) for k in self.expected} + actual = {k: gensim.models.utils_any2vec._ft_hash_broken_cy(k) for k in self.expected} self.assertEqual(self.expected_broken, actual) +# +# Run with: +# +# python -c 'import gensim.test.test_utils as t;t.ngram_main()' py_text 3 5 +# +def ngram_main(): + """Generate ngrams for tests from standard input.""" + import sys + import six + + alg = sys.argv[1] + minn = int(sys.argv[2]) + maxn = int(sys.argv[3]) + + assert six.PY3, 'this only works under Py3' + assert not DISABLE_CYTHON_TESTS, 'this only works if Cython extensions available' + assert minn <= maxn, 'expected sane command-line parameters' + + hashmap = { + 'py_text': gensim.models.utils_any2vec._compute_ngrams_py, + 'py_bytes': gensim.models.utils_any2vec._compute_ngrams_bytes_py, + 'cy_text': gensim.models.utils_any2vec._compute_ngrams_cy, + 'cy_bytes': gensim.models.utils_any2vec._compute_ngrams_bytes_cy, + } + try: + fun = hashmap[alg] + except KeyError: + raise KeyError('invalid alg: %r expected one of %r' % (alg, sorted(hashmap))) + + for line in sys.stdin: + word = line.rstrip('\n') + ngrams = fun(word, minn, maxn) + print("%r: %r," % (word, ngrams)) + + +class NgramsTest(unittest.TestCase): + def setUp(self): + self.expected_text = { + 'test': ['', '', ''], + 'at the': [ + '', + '', '' + ], + 'at\nthe': [ + '', + '', '' + ], + 'тест': ['<те', 'тес', 'ест', 'ст>', '<тес', 'тест', 'ест>', '<тест', 'тест>'], + 'テスト': ['<テス', 'テスト', 'スト>', '<テスト', 'テスト>', '<テスト>'], + '試し': ['<試し', '試し>', '<試し>'], + } + self.expected_bytes = { + 'test': [b'', b'est', b'est>', b'st>'], + 'at the': [ + b'', b'the', b'the>', b'he>' + ], + 'тест': [ + b'<\xd1\x82\xd0\xb5', b'<\xd1\x82\xd0\xb5\xd1\x81', b'<\xd1\x82\xd0\xb5\xd1\x81\xd1\x82', + b'\xd1\x82\xd0\xb5\xd1\x81', b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82', b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82>', + b'\xd0\xb5\xd1\x81\xd1\x82', b'\xd0\xb5\xd1\x81\xd1\x82>', b'\xd1\x81\xd1\x82>' + ], + 'テスト': [ + b'<\xe3\x83\x86\xe3\x82\xb9', b'<\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88', + b'<\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88>', b'\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88', + b'\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88>', b'\xe3\x82\xb9\xe3\x83\x88>' + ], + '試し': [b'<\xe8\xa9\xa6\xe3\x81\x97', b'<\xe8\xa9\xa6\xe3\x81\x97>', b'\xe8\xa9\xa6\xe3\x81\x97>'], + } + + self.expected_text_wide_unicode = { + '🚑🚒🚓🚕': [ + '<🚑🚒', '🚑🚒🚓', '🚒🚓🚕', '🚓🚕>', + '<🚑🚒🚓', '🚑🚒🚓🚕', '🚒🚓🚕>', '<🚑🚒🚓🚕', '🚑🚒🚓🚕>' + ], + } + self.expected_bytes_wide_unicode = { + '🚑🚒🚓🚕': [ + b'<\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92', + b'<\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93', + b'<\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95', + b'\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93', + b'\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95', + b'\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95>', + b'\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95', + b'\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95>', + b'\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95>' + ], + } + + def test_text_py(self): + for word in self.expected_text: + expected = self.expected_text[word] + actual = gensim.models.utils_any2vec._compute_ngrams_py(word, 3, 5) + self.assertEqual(expected, actual) + + @unittest.skipIf(sys.maxunicode == 0xffff, "Python interpreter doesn't support UCS-4 (wide unicode)") + def test_text_py_wide_unicode(self): + for word in self.expected_text_wide_unicode: + expected = self.expected_text_wide_unicode[word] + actual = gensim.models.utils_any2vec._compute_ngrams_py(word, 3, 5) + self.assertEqual(expected, actual) + + @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') + def test_text_cy(self): + for word in self.expected_text: + expected = self.expected_text[word] + actual = gensim.models.utils_any2vec._compute_ngrams_cy(word, 3, 5) + self.assertEqual(expected, actual) + + @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') + @unittest.skipIf(sys.maxunicode == 0xffff, "Python interpreter doesn't support UCS-4 (wide unicode)") + def test_text_cy_wide_unicode(self): + for word in self.expected_text_wide_unicode: + expected = self.expected_text_wide_unicode[word] + actual = gensim.models.utils_any2vec._compute_ngrams_cy(word, 3, 5) + self.assertEqual(expected, actual) + + def test_bytes_py(self): + for word in self.expected_bytes: + expected = self.expected_bytes[word] + actual = gensim.models.utils_any2vec._compute_ngrams_bytes_py(word, 3, 5) + self.assertEqual(expected, actual) + + expected_text = self.expected_text[word] + actual_text = [n.decode('utf-8') for n in actual] + # + # The text and byte implementations yield ngrams in different + # order, so the test ignores ngram order. + # + self.assertEqual(sorted(expected_text), sorted(actual_text)) + + for word in self.expected_bytes_wide_unicode: + expected = self.expected_bytes_wide_unicode[word] + actual = gensim.models.utils_any2vec._compute_ngrams_bytes_py(word, 3, 5) + self.assertEqual(expected, actual) + + expected_text = self.expected_text_wide_unicode[word] + actual_text = [n.decode('utf-8') for n in actual] + + self.assertEqual(sorted(expected_text), sorted(actual_text)) + + @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') + def test_bytes_cy(self): + for word in self.expected_bytes: + expected = self.expected_bytes[word] + actual = gensim.models.utils_any2vec._compute_ngrams_bytes_cy(word, 3, 5) + self.assertEqual(expected, actual) + + expected_text = self.expected_text[word] + actual_text = [n.decode('utf-8') for n in actual] + self.assertEqual(sorted(expected_text), sorted(actual_text)) + + for word in self.expected_bytes_wide_unicode: + expected = self.expected_bytes_wide_unicode[word] + actual = gensim.models.utils_any2vec._compute_ngrams_bytes_cy(word, 3, 5) + self.assertEqual(expected, actual) + + expected_text = self.expected_text_wide_unicode[word] + actual_text = [n.decode('utf-8') for n in actual] + self.assertEqual(sorted(expected_text), sorted(actual_text)) + + def test_fb(self): + """Test against results from Facebook's implementation.""" + with smart_open.smart_open(datapath('fb-ngrams.txt'), 'r', encoding='utf-8') as fin: + fb = dict(_read_fb(fin)) + + for word, expected in fb.items(): + # + # The model was trained with minn=3, maxn=6 + # + actual = gensim.models.utils_any2vec._compute_ngrams_py(word, 3, 6) + self.assertEqual(sorted(expected), sorted(actual)) + + +def _read_fb(fin): + """Read ngrams from output of the FB utility.""" + # + # $ cat words.txt + # test + # at the + # at\nthe + # тест + # テスト + # 試し + # 🚑🚒🚓🚕 + # $ while read w; + # do + # echo ""; + # echo $w; + # ./fasttext print-ngrams gensim/test/test_data/crime-and-punishment.bin "$w"; + # echo ""; + # done < words.txt > gensim/test/test_data/fb-ngrams.txt + # + while fin: + line = fin.readline().rstrip() + if not line: + break + + assert line == '' + word = fin.readline().rstrip() + + fin.readline() # ignore this line, it contains an origin vector for the full term + + ngrams = [] + while True: + line = fin.readline().rstrip() + if line == '': + break + + columns = line.split(' ') + term = ' '.join(columns[:-5]) + ngrams.append(term) + + yield word, ngrams + + if __name__ == '__main__': logging.root.setLevel(logging.WARNING) unittest.main()