Generated by Cython 0.29.5
Yellow lines hint at Python interaction.
Click on a line that starts with a "+" to see the C code that Cython generated for it.
Raw output: rolling_checksum_pyx_mod.c
001:
002: # cython: language_level=3, boundscheck=False
003:
004: """Provides a very simple rolling checksum for file data."""
005:
+006: import sys
__pyx_t_1 = __Pyx_Import(__pyx_n_s_sys, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 6, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_d, __pyx_n_s_sys, __pyx_t_1) < 0) __PYX_ERR(0, 6, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
007: from libc.stdint cimport uint64_t
008:
009:
+010: def min_max_chunker(file_):
/* Python wrapper */
static PyObject *__pyx_pw_24rolling_checksum_pyx_mod_1min_max_chunker(PyObject *__pyx_self, PyObject *__pyx_v_file_); /*proto*/
static char __pyx_doc_24rolling_checksum_pyx_mod_min_max_chunker[] = "Make sure chunk sizes are above and below a pair of thresholds.";
static PyMethodDef __pyx_mdef_24rolling_checksum_pyx_mod_1min_max_chunker = {"min_max_chunker", (PyCFunction)__pyx_pw_24rolling_checksum_pyx_mod_1min_max_chunker, METH_O, __pyx_doc_24rolling_checksum_pyx_mod_min_max_chunker};
static PyObject *__pyx_pw_24rolling_checksum_pyx_mod_1min_max_chunker(PyObject *__pyx_self, PyObject *__pyx_v_file_) {
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("min_max_chunker (wrapper)", 0);
__pyx_r = __pyx_pf_24rolling_checksum_pyx_mod_min_max_chunker(__pyx_self, ((PyObject *)__pyx_v_file_));
/* function exit code */
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_24rolling_checksum_pyx_mod_min_max_chunker(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_file_) {
struct __pyx_obj_24rolling_checksum_pyx_mod___pyx_scope_struct__min_max_chunker *__pyx_cur_scope;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("min_max_chunker", 0);
__pyx_cur_scope = (struct __pyx_obj_24rolling_checksum_pyx_mod___pyx_scope_struct__min_max_chunker *)__pyx_tp_new_24rolling_checksum_pyx_mod___pyx_scope_struct__min_max_chunker(__pyx_ptype_24rolling_checksum_pyx_mod___pyx_scope_struct__min_max_chunker, __pyx_empty_tuple, NULL);
if (unlikely(!__pyx_cur_scope)) {
__pyx_cur_scope = ((struct __pyx_obj_24rolling_checksum_pyx_mod___pyx_scope_struct__min_max_chunker *)Py_None);
__Pyx_INCREF(Py_None);
__PYX_ERR(0, 10, __pyx_L1_error)
} else {
__Pyx_GOTREF(__pyx_cur_scope);
}
__pyx_cur_scope->__pyx_v_file_ = __pyx_v_file_;
__Pyx_INCREF(__pyx_cur_scope->__pyx_v_file_);
__Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_file_);
{
__pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_24rolling_checksum_pyx_mod_2generator, __pyx_codeobj_, (PyObject *) __pyx_cur_scope, __pyx_n_s_min_max_chunker, __pyx_n_s_min_max_chunker, __pyx_n_s_rolling_checksum_pyx_mod); if (unlikely(!gen)) __PYX_ERR(0, 10, __pyx_L1_error)
__Pyx_DECREF(__pyx_cur_scope);
__Pyx_RefNannyFinishContext();
return (PyObject *) gen;
}
/* function exit code */
__pyx_L1_error:;
__Pyx_AddTraceback("rolling_checksum_pyx_mod.min_max_chunker", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__Pyx_DECREF(((PyObject *)__pyx_cur_scope));
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_gb_24rolling_checksum_pyx_mod_2generator(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value) /* generator body */
{
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("min_max_chunker", 0);
__pyx_L3_first_run:;
if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 10, __pyx_L1_error)
/* … */
/* function exit code */
PyErr_SetNone(PyExc_StopIteration);
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_AddTraceback("min_max_chunker", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_L0:;
__Pyx_XDECREF(__pyx_r); __pyx_r = 0;
#if !CYTHON_USE_EXC_INFO_STACK
__Pyx_Coroutine_ResetAndClearException(__pyx_generator);
#endif
__pyx_generator->resume_label = -1;
__Pyx_Coroutine_clear((PyObject*)__pyx_generator);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_gb_24rolling_checksum_pyx_mod_5generator1(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value); /* proto */
/* … */
__pyx_tuple__5 = PyTuple_Pack(7, __pyx_n_s_file, __pyx_n_s_empty, __pyx_n_s_minimum_length, __pyx_n_s_maximum_length, __pyx_n_s_chunk, __pyx_n_s_chunk_so_far, __pyx_n_s_len_chunk_so_far); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(0, 10, __pyx_L1_error)
__Pyx_GOTREF(__pyx_tuple__5);
__Pyx_GIVEREF(__pyx_tuple__5);
/* … */
__pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_24rolling_checksum_pyx_mod_1min_max_chunker, NULL, __pyx_n_s_rolling_checksum_pyx_mod); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 10, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
if (PyDict_SetItem(__pyx_d, __pyx_n_s_min_max_chunker, __pyx_t_1) < 0) __PYX_ERR(0, 10, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
__pyx_codeobj_ = (PyObject*)__Pyx_PyCode_New(1, 0, 7, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__5, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_rolling_checksum_pyx_mod_pyx, __pyx_n_s_min_max_chunker, 10, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj_)) __PYX_ERR(0, 10, __pyx_L1_error)
/* … */
struct __pyx_obj_24rolling_checksum_pyx_mod___pyx_scope_struct__min_max_chunker {
PyObject_HEAD
PyObject *__pyx_v_chunk;
PyObject *__pyx_v_chunk_so_far;
int __pyx_v_empty;
PyObject *__pyx_v_file_;
Py_ssize_t __pyx_v_len_chunk_so_far;
long __pyx_v_maximum_length;
long __pyx_v_minimum_length;
PyObject *__pyx_t_0;
Py_ssize_t __pyx_t_1;
PyObject *(*__pyx_t_2)(PyObject *);
};
011: """Make sure chunk sizes are above and below a pair of thresholds."""
+012: empty = True
__pyx_cur_scope->__pyx_v_empty = 1;
+013: minimum_length = 2 ** 19 # 0.5 mebibytes
__pyx_cur_scope->__pyx_v_minimum_length = 0x80000;
+014: maximum_length = 2 ** 22 # 4.0 mebibytes
__pyx_cur_scope->__pyx_v_maximum_length = 0x400000;
015:
+016: for chunk in n_level_chunker(file_, levels=3):
__Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_n_level_chunker); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_file_); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_file_); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_cur_scope->__pyx_v_file_); __pyx_t_3 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (PyDict_SetItem(__pyx_t_3, __pyx_n_s_levels, __pyx_int_3) < 0) __PYX_ERR(0, 16, __pyx_L1_error) __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (likely(PyList_CheckExact(__pyx_t_4)) || PyTuple_CheckExact(__pyx_t_4)) { __pyx_t_3 = __pyx_t_4; __Pyx_INCREF(__pyx_t_3); __pyx_t_5 = 0; __pyx_t_6 = NULL; } else { __pyx_t_5 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_6 = Py_TYPE(__pyx_t_3)->tp_iternext; if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 16, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; for (;;) { if (likely(!__pyx_t_6)) { if (likely(PyList_CheckExact(__pyx_t_3))) { if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_3)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_4 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_5); __Pyx_INCREF(__pyx_t_4); __pyx_t_5++; if (unlikely(0 < 0)) __PYX_ERR(0, 16, __pyx_L1_error) #else __pyx_t_4 = PySequence_ITEM(__pyx_t_3, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); #endif } else { if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_3)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_4 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_5); __Pyx_INCREF(__pyx_t_4); __pyx_t_5++; if (unlikely(0 < 0)) __PYX_ERR(0, 16, __pyx_L1_error) #else __pyx_t_4 = PySequence_ITEM(__pyx_t_3, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); #endif } } else { __pyx_t_4 = __pyx_t_6(__pyx_t_3); if (unlikely(!__pyx_t_4)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); else __PYX_ERR(0, 16, __pyx_L1_error) } break; } __Pyx_GOTREF(__pyx_t_4); } __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_chunk); __Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_chunk, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; /* … */ __pyx_L4_continue:; } __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+017: if empty:
__pyx_t_7 = (__pyx_cur_scope->__pyx_v_empty != 0);
if (__pyx_t_7) {
/* … */
goto __pyx_L6;
}
+018: chunk_so_far = chunk
__Pyx_INCREF(__pyx_cur_scope->__pyx_v_chunk); __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_chunk_so_far); __Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_chunk_so_far, __pyx_cur_scope->__pyx_v_chunk); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_chunk);
019: else:
+020: chunk_so_far += chunk
/*else*/ {
if (unlikely(!__pyx_cur_scope->__pyx_v_chunk_so_far)) { __Pyx_RaiseUnboundLocalError("chunk_so_far"); __PYX_ERR(0, 20, __pyx_L1_error) }
__pyx_t_4 = PyNumber_InPlaceAdd(__pyx_cur_scope->__pyx_v_chunk_so_far, __pyx_cur_scope->__pyx_v_chunk); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 20, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_4);
__Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_chunk_so_far);
__Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_chunk_so_far, __pyx_t_4);
__Pyx_GIVEREF(__pyx_t_4);
__pyx_t_4 = 0;
}
__pyx_L6:;
+021: empty = False
__pyx_cur_scope->__pyx_v_empty = 0;
022:
+023: len_chunk_so_far = len(chunk_so_far)
__pyx_t_8 = PyObject_Length(__pyx_cur_scope->__pyx_v_chunk_so_far); if (unlikely(__pyx_t_8 == ((Py_ssize_t)-1))) __PYX_ERR(0, 23, __pyx_L1_error) __pyx_cur_scope->__pyx_v_len_chunk_so_far = __pyx_t_8;
024:
+025: if len_chunk_so_far < minimum_length:
__pyx_t_7 = ((__pyx_cur_scope->__pyx_v_len_chunk_so_far < __pyx_cur_scope->__pyx_v_minimum_length) != 0);
if (__pyx_t_7) {
/* … */
}
026: # go back up for another piece
+027: continue
goto __pyx_L4_continue;
028:
+029: if len_chunk_so_far < maximum_length:
__pyx_t_7 = ((__pyx_cur_scope->__pyx_v_len_chunk_so_far < __pyx_cur_scope->__pyx_v_maximum_length) != 0);
if (__pyx_t_7) {
/* … */
}
030: # good - we're in the sweet spot
+031: yield chunk_so_far
__Pyx_INCREF(__pyx_cur_scope->__pyx_v_chunk_so_far); __pyx_r = __pyx_cur_scope->__pyx_v_chunk_so_far; __Pyx_XGIVEREF(__pyx_t_3); __pyx_cur_scope->__pyx_t_0 = __pyx_t_3; __pyx_cur_scope->__pyx_t_1 = __pyx_t_5; __pyx_cur_scope->__pyx_t_2 = __pyx_t_6; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); __Pyx_Coroutine_ResetAndClearException(__pyx_generator); /* return from generator, yielding value */ __pyx_generator->resume_label = 1; return __pyx_r; __pyx_L9_resume_from_yield:; __pyx_t_3 = __pyx_cur_scope->__pyx_t_0; __pyx_cur_scope->__pyx_t_0 = 0; __Pyx_XGOTREF(__pyx_t_3); __pyx_t_5 = __pyx_cur_scope->__pyx_t_1; __pyx_t_6 = __pyx_cur_scope->__pyx_t_2; if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 31, __pyx_L1_error)
+032: empty = True
__pyx_cur_scope->__pyx_v_empty = 1;
+033: continue
goto __pyx_L4_continue;
034:
+035: if len_chunk_so_far >= maximum_length:
__pyx_t_7 = ((__pyx_cur_scope->__pyx_v_len_chunk_so_far >= __pyx_cur_scope->__pyx_v_maximum_length) != 0);
if (__pyx_t_7) {
/* … */
}
036: # we have a long chunk - split it and go around for more
+037: yield chunk_so_far[:maximum_length]
__pyx_t_4 = __Pyx_PyObject_GetSlice(__pyx_cur_scope->__pyx_v_chunk_so_far, 0, __pyx_cur_scope->__pyx_v_maximum_length, NULL, NULL, NULL, 0, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 37, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_r = __pyx_t_4; __pyx_t_4 = 0; __Pyx_XGIVEREF(__pyx_t_3); __pyx_cur_scope->__pyx_t_0 = __pyx_t_3; __pyx_cur_scope->__pyx_t_1 = __pyx_t_5; __pyx_cur_scope->__pyx_t_2 = __pyx_t_6; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); __Pyx_Coroutine_ResetAndClearException(__pyx_generator); /* return from generator, yielding value */ __pyx_generator->resume_label = 2; return __pyx_r; __pyx_L11_resume_from_yield:; __pyx_t_3 = __pyx_cur_scope->__pyx_t_0; __pyx_cur_scope->__pyx_t_0 = 0; __Pyx_XGOTREF(__pyx_t_3); __pyx_t_5 = __pyx_cur_scope->__pyx_t_1; __pyx_t_6 = __pyx_cur_scope->__pyx_t_2; if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 37, __pyx_L1_error)
+038: chunk_so_far = chunk_so_far[maximum_length:]
__pyx_t_4 = __Pyx_PyObject_GetSlice(__pyx_cur_scope->__pyx_v_chunk_so_far, __pyx_cur_scope->__pyx_v_maximum_length, 0, NULL, NULL, NULL, 1, 0, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 38, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GOTREF(__pyx_cur_scope->__pyx_v_chunk_so_far); __Pyx_DECREF_SET(__pyx_cur_scope->__pyx_v_chunk_so_far, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0;
+039: continue
goto __pyx_L4_continue;
040:
+041: raise AssertionError("Should never reach this point")
__pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_AssertionError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 41, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __PYX_ERR(0, 41, __pyx_L1_error) /* … */ __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_u_Should_never_reach_this_point); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(0, 41, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2);
042:
+043: if not empty and chunk_so_far:
__pyx_t_9 = ((!(__pyx_cur_scope->__pyx_v_empty != 0)) != 0);
if (__pyx_t_9) {
} else {
__pyx_t_7 = __pyx_t_9;
goto __pyx_L13_bool_binop_done;
}
if (unlikely(!__pyx_cur_scope->__pyx_v_chunk_so_far)) { __Pyx_RaiseUnboundLocalError("chunk_so_far"); __PYX_ERR(0, 43, __pyx_L1_error) }
__pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_chunk_so_far); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 43, __pyx_L1_error)
__pyx_t_7 = __pyx_t_9;
__pyx_L13_bool_binop_done:;
if (__pyx_t_7) {
/* … */
}
CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope);
+044: yield chunk_so_far
if (unlikely(!__pyx_cur_scope->__pyx_v_chunk_so_far)) { __Pyx_RaiseUnboundLocalError("chunk_so_far"); __PYX_ERR(0, 44, __pyx_L1_error) }
__Pyx_INCREF(__pyx_cur_scope->__pyx_v_chunk_so_far);
__pyx_r = __pyx_cur_scope->__pyx_v_chunk_so_far;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
__Pyx_Coroutine_ResetAndClearException(__pyx_generator);
/* return from generator, yielding value */
__pyx_generator->resume_label = 3;
return __pyx_r;
__pyx_L15_resume_from_yield:;
if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 44, __pyx_L1_error)
045:
046:
+047: def n_level_chunker(file_, int levels=1):
/* Python wrapper */
static PyObject *__pyx_pw_24rolling_checksum_pyx_mod_4n_level_chunker(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static char __pyx_doc_24rolling_checksum_pyx_mod_3n_level_chunker[] = "Divide a file up into portions when we get 3 in-range checksums in a row.";
static PyMethodDef __pyx_mdef_24rolling_checksum_pyx_mod_4n_level_chunker = {"n_level_chunker", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_24rolling_checksum_pyx_mod_4n_level_chunker, METH_VARARGS|METH_KEYWORDS, __pyx_doc_24rolling_checksum_pyx_mod_3n_level_chunker};
static PyObject *__pyx_pw_24rolling_checksum_pyx_mod_4n_level_chunker(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
PyObject *__pyx_v_file_ = 0;
int __pyx_v_levels;
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("n_level_chunker (wrapper)", 0);
{
static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_file,&__pyx_n_s_levels,0};
PyObject* values[2] = {0,0};
if (unlikely(__pyx_kwds)) {
Py_ssize_t kw_args;
const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
switch (pos_args) {
case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
CYTHON_FALLTHROUGH;
case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
CYTHON_FALLTHROUGH;
case 0: break;
default: goto __pyx_L5_argtuple_error;
}
kw_args = PyDict_Size(__pyx_kwds);
switch (pos_args) {
case 0:
if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_file)) != 0)) kw_args--;
else goto __pyx_L5_argtuple_error;
CYTHON_FALLTHROUGH;
case 1:
if (kw_args > 0) {
PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_levels);
if (value) { values[1] = value; kw_args--; }
}
}
if (unlikely(kw_args > 0)) {
if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "n_level_chunker") < 0)) __PYX_ERR(0, 47, __pyx_L3_error)
}
} else {
switch (PyTuple_GET_SIZE(__pyx_args)) {
case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
CYTHON_FALLTHROUGH;
case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
break;
default: goto __pyx_L5_argtuple_error;
}
}
__pyx_v_file_ = values[0];
if (values[1]) {
__pyx_v_levels = __Pyx_PyInt_As_int(values[1]); if (unlikely((__pyx_v_levels == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 47, __pyx_L3_error)
} else {
__pyx_v_levels = ((int)1);
}
}
goto __pyx_L4_argument_unpacking_done;
__pyx_L5_argtuple_error:;
__Pyx_RaiseArgtupleInvalid("n_level_chunker", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 47, __pyx_L3_error)
__pyx_L3_error:;
__Pyx_AddTraceback("rolling_checksum_pyx_mod.n_level_chunker", __pyx_clineno, __pyx_lineno, __pyx_filename);
__Pyx_RefNannyFinishContext();
return NULL;
__pyx_L4_argument_unpacking_done:;
__pyx_r = __pyx_pf_24rolling_checksum_pyx_mod_3n_level_chunker(__pyx_self, __pyx_v_file_, __pyx_v_levels);
/* function exit code */
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_24rolling_checksum_pyx_mod_3n_level_chunker(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_file_, int __pyx_v_levels) {
struct __pyx_obj_24rolling_checksum_pyx_mod___pyx_scope_struct_1_n_level_chunker *__pyx_cur_scope;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("n_level_chunker", 0);
__pyx_cur_scope = (struct __pyx_obj_24rolling_checksum_pyx_mod___pyx_scope_struct_1_n_level_chunker *)__pyx_tp_new_24rolling_checksum_pyx_mod___pyx_scope_struct_1_n_level_chunker(__pyx_ptype_24rolling_checksum_pyx_mod___pyx_scope_struct_1_n_level_chunker, __pyx_empty_tuple, NULL);
if (unlikely(!__pyx_cur_scope)) {
__pyx_cur_scope = ((struct __pyx_obj_24rolling_checksum_pyx_mod___pyx_scope_struct_1_n_level_chunker *)Py_None);
__Pyx_INCREF(Py_None);
__PYX_ERR(0, 47, __pyx_L1_error)
} else {
__Pyx_GOTREF(__pyx_cur_scope);
}
__pyx_cur_scope->__pyx_v_file_ = __pyx_v_file_;
__Pyx_INCREF(__pyx_cur_scope->__pyx_v_file_);
__Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_file_);
__pyx_cur_scope->__pyx_v_levels = __pyx_v_levels;
{
__pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_24rolling_checksum_pyx_mod_5generator1, __pyx_codeobj__3, (PyObject *) __pyx_cur_scope, __pyx_n_s_n_level_chunker, __pyx_n_s_n_level_chunker, __pyx_n_s_rolling_checksum_pyx_mod); if (unlikely(!gen)) __PYX_ERR(0, 47, __pyx_L1_error)
__Pyx_DECREF(__pyx_cur_scope);
__Pyx_RefNannyFinishContext();
return (PyObject *) gen;
}
/* function exit code */
__pyx_L1_error:;
__Pyx_AddTraceback("rolling_checksum_pyx_mod.n_level_chunker", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__Pyx_DECREF(((PyObject *)__pyx_cur_scope));
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_gb_24rolling_checksum_pyx_mod_5generator1(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value) /* generator body */
{
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("n_level_chunker", 0);
__pyx_L3_first_run:;
if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 47, __pyx_L1_error)
/* … */
/* function exit code */
PyErr_SetNone(PyExc_StopIteration);
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_AddTraceback("n_level_chunker", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_L0:;
__Pyx_XDECREF(__pyx_r); __pyx_r = 0;
#if !CYTHON_USE_EXC_INFO_STACK
__Pyx_Coroutine_ResetAndClearException(__pyx_generator);
#endif
__pyx_generator->resume_label = -1;
__Pyx_Coroutine_clear((PyObject*)__pyx_generator);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
__pyx_tuple__6 = PyTuple_Pack(16, __pyx_n_s_file, __pyx_n_s_levels, __pyx_n_s_byte, __pyx_n_s_modulus, __pyx_n_s_threshold, __pyx_n_s_consecutive_boundaries, __pyx_n_s_preferred_block_len, __pyx_n_s_need_ord, __pyx_n_s_first_time, __pyx_n_s_byteno, __pyx_n_s_rolling_checksum, __pyx_n_s_two_to_the_22nd, __pyx_n_s_block, __pyx_n_s_character, __pyx_n_s_checksum, __pyx_n_s_result); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(0, 47, __pyx_L1_error)
__Pyx_GOTREF(__pyx_tuple__6);
__Pyx_GIVEREF(__pyx_tuple__6);
/* … */
__pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_24rolling_checksum_pyx_mod_4n_level_chunker, NULL, __pyx_n_s_rolling_checksum_pyx_mod); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 47, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
if (PyDict_SetItem(__pyx_d, __pyx_n_s_n_level_chunker, __pyx_t_1) < 0) __PYX_ERR(0, 47, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
__pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(2, 0, 16, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__6, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_rolling_checksum_pyx_mod_pyx, __pyx_n_s_n_level_chunker, 47, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(0, 47, __pyx_L1_error)
/* … */
struct __pyx_obj_24rolling_checksum_pyx_mod___pyx_scope_struct_1_n_level_chunker {
PyObject_HEAD
PyObject *__pyx_v_block;
uint64_t __pyx_v_byte;
int __pyx_v_byteno;
PyObject *__pyx_v_character;
PyObject *__pyx_v_checksum;
int __pyx_v_consecutive_boundaries;
PyObject *__pyx_v_file_;
int __pyx_v_first_time;
int __pyx_v_levels;
uint64_t __pyx_v_modulus;
int __pyx_v_need_ord;
int __pyx_v_preferred_block_len;
PyObject *__pyx_v_result;
struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *__pyx_v_rolling_checksum;
float __pyx_v_threshold;
long __pyx_v_two_to_the_22nd;
PyObject *__pyx_t_0;
Py_ssize_t __pyx_t_1;
int __pyx_t_2;
PyObject *(*__pyx_t_3)(PyObject *);
};
048: """Divide a file up into portions when we get 3 in-range checksums in a row."""
049: # pylint: disable=R0912,R0914
050: # R0912: I looked into reducing the branch count here, but it seemed to make things more complex, not less
051: # R0914: I believe we need a few local variables
052:
053: cdef uint64_t byte
054: cdef uint64_t modulus
055: cdef float threshold
056: cdef int consecutive_boundaries
057: cdef int preferred_block_len
058: cdef int need_ord
059: cdef int first_time
060: cdef int byteno
061:
+062: rolling_checksum = Rolling_checksum()
__pyx_t_1 = __Pyx_PyObject_CallNoArg(((PyObject *)__pyx_ptype_24rolling_checksum_pyx_mod_Rolling_checksum)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 62, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_GIVEREF(__pyx_t_1); __pyx_cur_scope->__pyx_v_rolling_checksum = ((struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *)__pyx_t_1); __pyx_t_1 = 0;
063:
064: # this threshold should give us about 2**20 byte chunks on average
+065: modulus = rolling_checksum.get_modulus()
__pyx_t_1 = ((struct __pyx_vtabstruct_24rolling_checksum_pyx_mod_Rolling_checksum *)__pyx_cur_scope->__pyx_v_rolling_checksum->__pyx_vtab)->get_modulus(__pyx_cur_scope->__pyx_v_rolling_checksum, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 65, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __Pyx_PyInt_As_uint64_t(__pyx_t_1); if (unlikely((__pyx_t_2 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 65, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_cur_scope->__pyx_v_modulus = __pyx_t_2;
+066: threshold = modulus * 0.007
__pyx_cur_scope->__pyx_v_threshold = (__pyx_cur_scope->__pyx_v_modulus * 0.007);
067:
+068: consecutive_boundaries = 0
__pyx_cur_scope->__pyx_v_consecutive_boundaries = 0;
069:
+070: two_to_the_22nd = 2 ** 22
__pyx_cur_scope->__pyx_v_two_to_the_22nd = 0x400000;
+071: preferred_block_len = two_to_the_22nd
__pyx_cur_scope->__pyx_v_preferred_block_len = __pyx_cur_scope->__pyx_v_two_to_the_22nd;
072:
+073: block = file_.read(preferred_block_len)
__pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_cur_scope->__pyx_v_file_, __pyx_n_s_read); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 73, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = __Pyx_PyInt_From_int(__pyx_cur_scope->__pyx_v_preferred_block_len); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 73, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_3); if (likely(__pyx_t_5)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); __Pyx_INCREF(__pyx_t_5); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_3, function); } } __pyx_t_1 = (__pyx_t_5) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_5, __pyx_t_4) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_4); __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 73, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GIVEREF(__pyx_t_1); __pyx_cur_scope->__pyx_v_block = __pyx_t_1; __pyx_t_1 = 0;
074:
+075: while True:
while (1) {
+076: if not block[preferred_block_len - 1:]:
__pyx_t_1 = __Pyx_PyObject_GetSlice(__pyx_cur_scope->__pyx_v_block, (__pyx_cur_scope->__pyx_v_preferred_block_len - 1), 0, NULL, NULL, NULL, 1, 0, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 76, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(0, 76, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_7 = ((!__pyx_t_6) != 0); if (__pyx_t_7) { /* … */ }
+077: block = block + file_.read(preferred_block_len - len(block))
__pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_cur_scope->__pyx_v_file_, __pyx_n_s_read); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 77, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_8 = PyObject_Length(__pyx_cur_scope->__pyx_v_block); if (unlikely(__pyx_t_8 == ((Py_ssize_t)-1))) __PYX_ERR(0, 77, __pyx_L1_error) __pyx_t_4 = PyInt_FromSsize_t((__pyx_cur_scope->__pyx_v_preferred_block_len - __pyx_t_8)); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 77, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_3); if (likely(__pyx_t_5)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); __Pyx_INCREF(__pyx_t_5); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_3, function); } } __pyx_t_1 = (__pyx_t_5) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_5, __pyx_t_4) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_4); __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 77, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = PyNumber_Add(__pyx_cur_scope->__pyx_v_block, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 77, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GOTREF(__pyx_cur_scope->__pyx_v_block); __Pyx_DECREF_SET(__pyx_cur_scope->__pyx_v_block, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0;
078:
+079: if not block:
__pyx_t_7 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_block); if (unlikely(__pyx_t_7 < 0)) __PYX_ERR(0, 79, __pyx_L1_error) __pyx_t_6 = ((!__pyx_t_7) != 0); if (__pyx_t_6) { /* … */ }
+080: break
goto __pyx_L5_break;
081:
+082: for byteno, character in enumerate(block):
__pyx_t_9 = 0;
if (likely(PyList_CheckExact(__pyx_cur_scope->__pyx_v_block)) || PyTuple_CheckExact(__pyx_cur_scope->__pyx_v_block)) {
__pyx_t_3 = __pyx_cur_scope->__pyx_v_block; __Pyx_INCREF(__pyx_t_3); __pyx_t_8 = 0;
__pyx_t_10 = NULL;
} else {
__pyx_t_8 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_cur_scope->__pyx_v_block); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 82, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__pyx_t_10 = Py_TYPE(__pyx_t_3)->tp_iternext; if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 82, __pyx_L1_error)
}
for (;;) {
if (likely(!__pyx_t_10)) {
if (likely(PyList_CheckExact(__pyx_t_3))) {
if (__pyx_t_8 >= PyList_GET_SIZE(__pyx_t_3)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_1 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_8); __Pyx_INCREF(__pyx_t_1); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 82, __pyx_L1_error)
#else
__pyx_t_1 = PySequence_ITEM(__pyx_t_3, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 82, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
#endif
} else {
if (__pyx_t_8 >= PyTuple_GET_SIZE(__pyx_t_3)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_8); __Pyx_INCREF(__pyx_t_1); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 82, __pyx_L1_error)
#else
__pyx_t_1 = PySequence_ITEM(__pyx_t_3, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 82, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
#endif
}
} else {
__pyx_t_1 = __pyx_t_10(__pyx_t_3);
if (unlikely(!__pyx_t_1)) {
PyObject* exc_type = PyErr_Occurred();
if (exc_type) {
if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
else __PYX_ERR(0, 82, __pyx_L1_error)
}
break;
}
__Pyx_GOTREF(__pyx_t_1);
}
__Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_character);
__Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_character, __pyx_t_1);
__Pyx_GIVEREF(__pyx_t_1);
__pyx_t_1 = 0;
__pyx_cur_scope->__pyx_v_byteno = __pyx_t_9;
__pyx_t_9 = (__pyx_t_9 + 1);
/* … */
__pyx_L8_continue:;
}
/*else*/ {
/* … */
__pyx_L9_break:;
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
}
__pyx_L5_break:;
+083: byte = character
__pyx_t_2 = __Pyx_PyInt_As_uint64_t(__pyx_cur_scope->__pyx_v_character); if (unlikely((__pyx_t_2 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 83, __pyx_L1_error) __pyx_cur_scope->__pyx_v_byte = __pyx_t_2;
084:
+085: checksum = rolling_checksum.add_byte(byte)
__pyx_t_1 = ((struct __pyx_vtabstruct_24rolling_checksum_pyx_mod_Rolling_checksum *)__pyx_cur_scope->__pyx_v_rolling_checksum->__pyx_vtab)->add_byte(__pyx_cur_scope->__pyx_v_rolling_checksum, __pyx_cur_scope->__pyx_v_byte, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 85, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_checksum); __Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_checksum, __pyx_t_1); __Pyx_GIVEREF(__pyx_t_1); __pyx_t_1 = 0;
086:
+087: if checksum < threshold:
__pyx_t_1 = PyFloat_FromDouble(__pyx_cur_scope->__pyx_v_threshold); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 87, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_4 = PyObject_RichCompare(__pyx_cur_scope->__pyx_v_checksum, __pyx_t_1, Py_LT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 87, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(0, 87, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_6) { /* … */ goto __pyx_L10; }
+088: consecutive_boundaries += 1
__pyx_cur_scope->__pyx_v_consecutive_boundaries = (__pyx_cur_scope->__pyx_v_consecutive_boundaries + 1);
089: else:
+090: consecutive_boundaries = 0
/*else*/ {
__pyx_cur_scope->__pyx_v_consecutive_boundaries = 0;
+091: continue
goto __pyx_L8_continue;
}
__pyx_L10:;
092:
+093: if consecutive_boundaries == levels:
__pyx_t_6 = ((__pyx_cur_scope->__pyx_v_consecutive_boundaries == __pyx_cur_scope->__pyx_v_levels) != 0);
if (__pyx_t_6) {
/* … */
}
+094: consecutive_boundaries = 0
__pyx_cur_scope->__pyx_v_consecutive_boundaries = 0;
+095: result = block[:byteno]
__pyx_t_4 = __Pyx_PyObject_GetSlice(__pyx_cur_scope->__pyx_v_block, 0, __pyx_cur_scope->__pyx_v_byteno, NULL, NULL, NULL, 0, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_result); __Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_result, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0;
+096: block = block[byteno:]
__pyx_t_4 = __Pyx_PyObject_GetSlice(__pyx_cur_scope->__pyx_v_block, __pyx_cur_scope->__pyx_v_byteno, 0, NULL, NULL, NULL, 1, 0, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 96, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GOTREF(__pyx_cur_scope->__pyx_v_block); __Pyx_DECREF_SET(__pyx_cur_scope->__pyx_v_block, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0;
097:
+098: if result:
__pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_result); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(0, 98, __pyx_L1_error) if (__pyx_t_6) { /* … */ }
099: # result is not empty, so yield it
+100: yield result
__Pyx_INCREF(__pyx_cur_scope->__pyx_v_result); __pyx_r = __pyx_cur_scope->__pyx_v_result; __Pyx_XGIVEREF(__pyx_t_3); __pyx_cur_scope->__pyx_t_0 = __pyx_t_3; __pyx_cur_scope->__pyx_t_1 = __pyx_t_8; __pyx_cur_scope->__pyx_t_2 = __pyx_t_9; __pyx_cur_scope->__pyx_t_3 = __pyx_t_10; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); __Pyx_Coroutine_ResetAndClearException(__pyx_generator); /* return from generator, yielding value */ __pyx_generator->resume_label = 1; return __pyx_r; __pyx_L13_resume_from_yield:; __pyx_t_3 = __pyx_cur_scope->__pyx_t_0; __pyx_cur_scope->__pyx_t_0 = 0; __Pyx_XGOTREF(__pyx_t_3); __pyx_t_8 = __pyx_cur_scope->__pyx_t_1; __pyx_t_9 = __pyx_cur_scope->__pyx_t_2; __pyx_t_10 = __pyx_cur_scope->__pyx_t_3; if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 100, __pyx_L1_error)
101: # we must break out of the loop to restart byteno at 0
+102: break
goto __pyx_L9_break;
103: else:
104: # We made it all the way through the enumeration without yielding anything, so yield all we have and empty block
+105: if block:
__pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_block); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(0, 105, __pyx_L1_error) if (__pyx_t_6) { /* … */ } }
+106: yield block
__Pyx_INCREF(__pyx_cur_scope->__pyx_v_block); __pyx_r = __pyx_cur_scope->__pyx_v_block; __Pyx_XGIVEREF(__pyx_t_3); __pyx_cur_scope->__pyx_t_0 = __pyx_t_3; __pyx_cur_scope->__pyx_t_1 = __pyx_t_8; __pyx_cur_scope->__pyx_t_2 = __pyx_t_9; __pyx_cur_scope->__pyx_t_3 = __pyx_t_10; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); __Pyx_Coroutine_ResetAndClearException(__pyx_generator); /* return from generator, yielding value */ __pyx_generator->resume_label = 2; return __pyx_r; __pyx_L16_resume_from_yield:; __pyx_t_3 = __pyx_cur_scope->__pyx_t_0; __pyx_cur_scope->__pyx_t_0 = 0; __Pyx_XGOTREF(__pyx_t_3); __pyx_t_8 = __pyx_cur_scope->__pyx_t_1; __pyx_t_9 = __pyx_cur_scope->__pyx_t_2; __pyx_t_10 = __pyx_cur_scope->__pyx_t_3; if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 106, __pyx_L1_error)
107: # in other words, make block an empty string
+108: block = block[0:0]
__pyx_t_4 = __Pyx_PyObject_GetSlice(__pyx_cur_scope->__pyx_v_block, 0, 0, NULL, NULL, &__pyx_slice__4, 1, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 108, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GOTREF(__pyx_cur_scope->__pyx_v_block); __Pyx_DECREF_SET(__pyx_cur_scope->__pyx_v_block, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; /* … */ __pyx_slice__4 = PySlice_New(__pyx_int_0, __pyx_int_0, Py_None); if (unlikely(!__pyx_slice__4)) __PYX_ERR(0, 108, __pyx_L1_error) __Pyx_GOTREF(__pyx_slice__4); __Pyx_GIVEREF(__pyx_slice__4);
109:
110: # we're done processing chunks - return what's left, if it's not empty
+111: if block:
__pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_block); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(0, 111, __pyx_L1_error) if (__pyx_t_6) { /* … */ } CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope);
112: # result is not empty, so yield it
+113: sys.stderr.write('yielding block (3) of length %d\n' % len(result))
__Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_sys); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 113, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_stderr); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 113, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_write); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 113, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (unlikely(!__pyx_cur_scope->__pyx_v_result)) { __Pyx_RaiseUnboundLocalError("result"); __PYX_ERR(0, 113, __pyx_L1_error) } __pyx_t_8 = PyObject_Length(__pyx_cur_scope->__pyx_v_result); if (unlikely(__pyx_t_8 == ((Py_ssize_t)-1))) __PYX_ERR(0, 113, __pyx_L1_error) __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_8); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 113, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_5 = PyUnicode_Format(__pyx_kp_u_yielding_block_3_of_length_d, __pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 113, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_1)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); __Pyx_INCREF(__pyx_t_1); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_4, function); } } __pyx_t_3 = (__pyx_t_1) ? __Pyx_PyObject_Call2Args(__pyx_t_4, __pyx_t_1, __pyx_t_5) : __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_5); __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 113, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+114: yield block
__Pyx_INCREF(__pyx_cur_scope->__pyx_v_block); __pyx_r = __pyx_cur_scope->__pyx_v_block; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); __Pyx_Coroutine_ResetAndClearException(__pyx_generator); /* return from generator, yielding value */ __pyx_generator->resume_label = 3; return __pyx_r; __pyx_L18_resume_from_yield:; if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 114, __pyx_L1_error)
+115: block = file_.read(two_to_the_22nd)
__pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_cur_scope->__pyx_v_file_, __pyx_n_s_read); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 115, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_cur_scope->__pyx_v_two_to_the_22nd); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 115, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_1 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_1)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); __Pyx_INCREF(__pyx_t_1); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_4, function); } } __pyx_t_3 = (__pyx_t_1) ? __Pyx_PyObject_Call2Args(__pyx_t_4, __pyx_t_1, __pyx_t_5) : __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_5); __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 115, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_cur_scope->__pyx_v_block); __Pyx_DECREF_SET(__pyx_cur_scope->__pyx_v_block, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0;
+116: assert not block
#ifndef CYTHON_WITHOUT_ASSERTIONS
if (unlikely(!Py_OptimizeFlag)) {
__pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_block); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(0, 116, __pyx_L1_error)
if (unlikely(!((!__pyx_t_6) != 0))) {
PyErr_SetNone(PyExc_AssertionError);
__PYX_ERR(0, 116, __pyx_L1_error)
}
}
#endif
117:
118:
119: # This one actually seems to be better as an iterator than a generator
+120: cdef class Rolling_checksum(object):
struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum {
PyObject_HEAD
struct __pyx_vtabstruct_24rolling_checksum_pyx_mod_Rolling_checksum *__pyx_vtab;
uint64_t _total;
uint64_t _width;
uint64_t _multiplicand;
uint64_t _modulus;
uint64_t _addend;
uint64_t _offset;
PyObject *_window;
uint64_t _magic_checksum;
int first_time;
int need_ord;
};
/* … */
struct __pyx_vtabstruct_24rolling_checksum_pyx_mod_Rolling_checksum {
PyObject *(*add_byte)(struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *, uint64_t, int __pyx_skip_dispatch);
PyObject *(*get_modulus)(struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *, int __pyx_skip_dispatch);
};
static struct __pyx_vtabstruct_24rolling_checksum_pyx_mod_Rolling_checksum *__pyx_vtabptr_24rolling_checksum_pyx_mod_Rolling_checksum;
121: cdef uint64_t _total
122: cdef uint64_t _width
123: cdef uint64_t _multiplicand
124: cdef uint64_t _modulus
125: cdef uint64_t _addend
126: cdef uint64_t _offset
127: cdef list _window
128: cdef uint64_t _magic_checksum
129: cdef int first_time
130: cdef int need_ord
131: # pylint: disable=R0902
132: # R0902: We need a few instance attributes; we're a nontrivial iterator - but I suspect this'll beat polynomials bigtime
133: """
134: Compute a very simple, fast, rolling checksum.
135:
136: We don't maintain a list of bytes or anything - we just produce checksums on demand. Inspired by linear congruential
137: random number generation. It's not quite the same thing though.
138: """
139:
+140: def __init__(self, width=607, multiplicand=7, modulus=3677, addend=66041):
/* Python wrapper */
static int __pyx_pw_24rolling_checksum_pyx_mod_16Rolling_checksum_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static char __pyx_doc_24rolling_checksum_pyx_mod_16Rolling_checksum___init__[] = "Initialize.";
#if CYTHON_COMPILING_IN_CPYTHON
struct wrapperbase __pyx_wrapperbase_24rolling_checksum_pyx_mod_16Rolling_checksum___init__;
#endif
static int __pyx_pw_24rolling_checksum_pyx_mod_16Rolling_checksum_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
PyObject *__pyx_v_width = 0;
PyObject *__pyx_v_multiplicand = 0;
PyObject *__pyx_v_modulus = 0;
PyObject *__pyx_v_addend = 0;
int __pyx_r;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("__init__ (wrapper)", 0);
{
static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_width,&__pyx_n_s_multiplicand,&__pyx_n_s_modulus,&__pyx_n_s_addend,0};
PyObject* values[4] = {0,0,0,0};
values[0] = ((PyObject *)__pyx_int_607);
values[1] = ((PyObject *)__pyx_int_7);
values[2] = ((PyObject *)__pyx_int_3677);
values[3] = ((PyObject *)__pyx_int_66041);
if (unlikely(__pyx_kwds)) {
Py_ssize_t kw_args;
const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
switch (pos_args) {
case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3);
CYTHON_FALLTHROUGH;
case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
CYTHON_FALLTHROUGH;
case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
CYTHON_FALLTHROUGH;
case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
CYTHON_FALLTHROUGH;
case 0: break;
default: goto __pyx_L5_argtuple_error;
}
kw_args = PyDict_Size(__pyx_kwds);
switch (pos_args) {
case 0:
if (kw_args > 0) {
PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_width);
if (value) { values[0] = value; kw_args--; }
}
CYTHON_FALLTHROUGH;
case 1:
if (kw_args > 0) {
PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_multiplicand);
if (value) { values[1] = value; kw_args--; }
}
CYTHON_FALLTHROUGH;
case 2:
if (kw_args > 0) {
PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_modulus);
if (value) { values[2] = value; kw_args--; }
}
CYTHON_FALLTHROUGH;
case 3:
if (kw_args > 0) {
PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_addend);
if (value) { values[3] = value; kw_args--; }
}
}
if (unlikely(kw_args > 0)) {
if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(0, 140, __pyx_L3_error)
}
} else {
switch (PyTuple_GET_SIZE(__pyx_args)) {
case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3);
CYTHON_FALLTHROUGH;
case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
CYTHON_FALLTHROUGH;
case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
CYTHON_FALLTHROUGH;
case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
CYTHON_FALLTHROUGH;
case 0: break;
default: goto __pyx_L5_argtuple_error;
}
}
__pyx_v_width = values[0];
__pyx_v_multiplicand = values[1];
__pyx_v_modulus = values[2];
__pyx_v_addend = values[3];
}
goto __pyx_L4_argument_unpacking_done;
__pyx_L5_argtuple_error:;
__Pyx_RaiseArgtupleInvalid("__init__", 0, 0, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 140, __pyx_L3_error)
__pyx_L3_error:;
__Pyx_AddTraceback("rolling_checksum_pyx_mod.Rolling_checksum.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
__Pyx_RefNannyFinishContext();
return -1;
__pyx_L4_argument_unpacking_done:;
__pyx_r = __pyx_pf_24rolling_checksum_pyx_mod_16Rolling_checksum___init__(((struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *)__pyx_v_self), __pyx_v_width, __pyx_v_multiplicand, __pyx_v_modulus, __pyx_v_addend);
/* function exit code */
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static int __pyx_pf_24rolling_checksum_pyx_mod_16Rolling_checksum___init__(struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *__pyx_v_self, PyObject *__pyx_v_width, PyObject *__pyx_v_multiplicand, PyObject *__pyx_v_modulus, PyObject *__pyx_v_addend) {
int __pyx_r;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("__init__", 0);
/* … */
/* function exit code */
__pyx_r = 0;
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_AddTraceback("rolling_checksum_pyx_mod.Rolling_checksum.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = -1;
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
141: """Initialize."""
142: # pylint: disable=R0913
143: # R0913: We need a few arguments
+144: self._total = addend * width
__pyx_t_1 = PyNumber_Multiply(__pyx_v_addend, __pyx_v_width); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 144, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __Pyx_PyInt_As_uint64_t(__pyx_t_1); if (unlikely((__pyx_t_2 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 144, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->_total = __pyx_t_2;
+145: self._width = width
__pyx_t_2 = __Pyx_PyInt_As_uint64_t(__pyx_v_width); if (unlikely((__pyx_t_2 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 145, __pyx_L1_error) __pyx_v_self->_width = __pyx_t_2;
+146: self._multiplicand = multiplicand
__pyx_t_2 = __Pyx_PyInt_As_uint64_t(__pyx_v_multiplicand); if (unlikely((__pyx_t_2 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 146, __pyx_L1_error) __pyx_v_self->_multiplicand = __pyx_t_2;
+147: self._modulus = modulus
__pyx_t_2 = __Pyx_PyInt_As_uint64_t(__pyx_v_modulus); if (unlikely((__pyx_t_2 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 147, __pyx_L1_error) __pyx_v_self->_modulus = __pyx_t_2;
+148: self._addend = addend
__pyx_t_2 = __Pyx_PyInt_As_uint64_t(__pyx_v_addend); if (unlikely((__pyx_t_2 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 148, __pyx_L1_error) __pyx_v_self->_addend = __pyx_t_2;
+149: self._offset = 0
__pyx_v_self->_offset = 0;
+150: self._window = [addend] * self._width
__pyx_t_1 = PyList_New(1 * (__pyx_v_self->_width)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 150, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); { Py_ssize_t __pyx_temp; for (__pyx_temp=0; __pyx_temp < __pyx_v_self->_width; __pyx_temp++) { __Pyx_INCREF(__pyx_v_addend); __Pyx_GIVEREF(__pyx_v_addend); PyList_SET_ITEM(__pyx_t_1, __pyx_temp, __pyx_v_addend); } } __Pyx_GIVEREF(__pyx_t_1); __Pyx_GOTREF(__pyx_v_self->_window); __Pyx_DECREF(__pyx_v_self->_window); __pyx_v_self->_window = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
151:
+152: cpdef add_byte(self, uint64_t byte):
static PyObject *__pyx_pw_24rolling_checksum_pyx_mod_16Rolling_checksum_3add_byte(PyObject *__pyx_v_self, PyObject *__pyx_arg_byte); /*proto*/
static PyObject *__pyx_f_24rolling_checksum_pyx_mod_16Rolling_checksum_add_byte(struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *__pyx_v_self, uint64_t __pyx_v_byte, int __pyx_skip_dispatch) {
PyObject *__pyx_v_adjusted = NULL;
uint64_t __pyx_v_new_offset;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("add_byte", 0);
/* Check if called by wrapper */
if (unlikely(__pyx_skip_dispatch)) ;
/* Check if overridden in Python */
else if (unlikely((Py_TYPE(((PyObject *)__pyx_v_self))->tp_dictoffset != 0) || (Py_TYPE(((PyObject *)__pyx_v_self))->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) {
#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP
static PY_UINT64_T __pyx_tp_dict_version = __PYX_DICT_VERSION_INIT, __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
if (unlikely(!__Pyx_object_dict_version_matches(((PyObject *)__pyx_v_self), __pyx_tp_dict_version, __pyx_obj_dict_version))) {
PY_UINT64_T __pyx_type_dict_guard = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
#endif
__pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_add_byte); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 152, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
if (!PyCFunction_Check(__pyx_t_1) || (PyCFunction_GET_FUNCTION(__pyx_t_1) != (PyCFunction)(void*)__pyx_pw_24rolling_checksum_pyx_mod_16Rolling_checksum_3add_byte)) {
__Pyx_XDECREF(__pyx_r);
__pyx_t_3 = __Pyx_PyInt_From_uint64_t(__pyx_v_byte); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 152, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_INCREF(__pyx_t_1);
__pyx_t_4 = __pyx_t_1; __pyx_t_5 = NULL;
if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_4))) {
__pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4);
if (likely(__pyx_t_5)) {
PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4);
__Pyx_INCREF(__pyx_t_5);
__Pyx_INCREF(function);
__Pyx_DECREF_SET(__pyx_t_4, function);
}
}
__pyx_t_2 = (__pyx_t_5) ? __Pyx_PyObject_Call2Args(__pyx_t_4, __pyx_t_5, __pyx_t_3) : __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3);
__Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 152, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
__pyx_r = __pyx_t_2;
__pyx_t_2 = 0;
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
goto __pyx_L0;
}
#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP
__pyx_tp_dict_version = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
__pyx_obj_dict_version = __Pyx_get_object_dict_version(((PyObject *)__pyx_v_self));
if (unlikely(__pyx_type_dict_guard != __pyx_tp_dict_version)) {
__pyx_tp_dict_version = __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
}
#endif
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP
}
#endif
}
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_AddTraceback("rolling_checksum_pyx_mod.Rolling_checksum.add_byte", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_adjusted);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* Python wrapper */
static PyObject *__pyx_pw_24rolling_checksum_pyx_mod_16Rolling_checksum_3add_byte(PyObject *__pyx_v_self, PyObject *__pyx_arg_byte); /*proto*/
static char __pyx_doc_24rolling_checksum_pyx_mod_16Rolling_checksum_2add_byte[] = "Add a byte into our rolling checksum function.";
static PyObject *__pyx_pw_24rolling_checksum_pyx_mod_16Rolling_checksum_3add_byte(PyObject *__pyx_v_self, PyObject *__pyx_arg_byte) {
uint64_t __pyx_v_byte;
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("add_byte (wrapper)", 0);
assert(__pyx_arg_byte); {
__pyx_v_byte = __Pyx_PyInt_As_uint64_t(__pyx_arg_byte); if (unlikely((__pyx_v_byte == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 152, __pyx_L3_error)
}
goto __pyx_L4_argument_unpacking_done;
__pyx_L3_error:;
__Pyx_AddTraceback("rolling_checksum_pyx_mod.Rolling_checksum.add_byte", __pyx_clineno, __pyx_lineno, __pyx_filename);
__Pyx_RefNannyFinishContext();
return NULL;
__pyx_L4_argument_unpacking_done:;
__pyx_r = __pyx_pf_24rolling_checksum_pyx_mod_16Rolling_checksum_2add_byte(((struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *)__pyx_v_self), ((uint64_t)__pyx_v_byte));
/* function exit code */
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_24rolling_checksum_pyx_mod_16Rolling_checksum_2add_byte(struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *__pyx_v_self, uint64_t __pyx_v_byte) {
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("add_byte", 0);
__Pyx_XDECREF(__pyx_r);
__pyx_t_1 = __pyx_f_24rolling_checksum_pyx_mod_16Rolling_checksum_add_byte(__pyx_v_self, __pyx_v_byte, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 152, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_r = __pyx_t_1;
__pyx_t_1 = 0;
goto __pyx_L0;
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_AddTraceback("rolling_checksum_pyx_mod.Rolling_checksum.add_byte", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
153: """Add a byte into our rolling checksum function."""
+154: adjusted = byte * self._multiplicand + self._addend
__pyx_t_1 = __Pyx_PyInt_From_uint64_t(((__pyx_v_byte * __pyx_v_self->_multiplicand) + __pyx_v_self->_addend)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_adjusted = __pyx_t_1; __pyx_t_1 = 0;
+155: self._window[self._offset] = adjusted
if (unlikely(__pyx_v_self->_window == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 155, __pyx_L1_error)
}
if (unlikely(__Pyx_SetItemInt(__pyx_v_self->_window, __pyx_v_self->_offset, __pyx_v_adjusted, uint64_t, 0, __Pyx_PyInt_From_uint64_t, 1, 0, 0) < 0)) __PYX_ERR(0, 155, __pyx_L1_error)
+156: self._total += adjusted
__pyx_t_1 = __Pyx_PyInt_From_uint64_t(__pyx_v_self->_total); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 156, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = PyNumber_InPlaceAdd(__pyx_t_1, __pyx_v_adjusted); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 156, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_6 = __Pyx_PyInt_As_uint64_t(__pyx_t_2); if (unlikely((__pyx_t_6 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 156, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_self->_total = __pyx_t_6;
+157: new_offset = (self._offset + 1) % self._width
__pyx_t_6 = (__pyx_v_self->_offset + 1);
if (unlikely(__pyx_v_self->_width == 0)) {
PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero");
__PYX_ERR(0, 157, __pyx_L1_error)
}
__pyx_v_new_offset = (__pyx_t_6 % __pyx_v_self->_width);
+158: self._offset = new_offset
__pyx_v_self->_offset = __pyx_v_new_offset;
+159: self._total -= self._window[new_offset]
__pyx_t_2 = __Pyx_PyInt_From_uint64_t(__pyx_v_self->_total); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 159, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (unlikely(__pyx_v_self->_window == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); __PYX_ERR(0, 159, __pyx_L1_error) } __pyx_t_1 = PyNumber_InPlaceSubtract(__pyx_t_2, PyList_GET_ITEM(__pyx_v_self->_window, __pyx_v_new_offset)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 159, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_6 = __Pyx_PyInt_As_uint64_t(__pyx_t_1); if (unlikely((__pyx_t_6 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 159, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->_total = __pyx_t_6;
+160: return self._total % self._modulus
__Pyx_XDECREF(__pyx_r); if (unlikely(__pyx_v_self->_modulus == 0)) { PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); __PYX_ERR(0, 160, __pyx_L1_error) } __pyx_t_1 = __Pyx_PyInt_From_uint64_t((__pyx_v_self->_total % __pyx_v_self->_modulus)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0;
161:
+162: cpdef get_modulus(self):
static PyObject *__pyx_pw_24rolling_checksum_pyx_mod_16Rolling_checksum_5get_modulus(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/
static PyObject *__pyx_f_24rolling_checksum_pyx_mod_16Rolling_checksum_get_modulus(struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *__pyx_v_self, int __pyx_skip_dispatch) {
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("get_modulus", 0);
/* Check if called by wrapper */
if (unlikely(__pyx_skip_dispatch)) ;
/* Check if overridden in Python */
else if (unlikely((Py_TYPE(((PyObject *)__pyx_v_self))->tp_dictoffset != 0) || (Py_TYPE(((PyObject *)__pyx_v_self))->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) {
#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP
static PY_UINT64_T __pyx_tp_dict_version = __PYX_DICT_VERSION_INIT, __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
if (unlikely(!__Pyx_object_dict_version_matches(((PyObject *)__pyx_v_self), __pyx_tp_dict_version, __pyx_obj_dict_version))) {
PY_UINT64_T __pyx_type_dict_guard = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
#endif
__pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_get_modulus); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 162, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
if (!PyCFunction_Check(__pyx_t_1) || (PyCFunction_GET_FUNCTION(__pyx_t_1) != (PyCFunction)(void*)__pyx_pw_24rolling_checksum_pyx_mod_16Rolling_checksum_5get_modulus)) {
__Pyx_XDECREF(__pyx_r);
__Pyx_INCREF(__pyx_t_1);
__pyx_t_3 = __pyx_t_1; __pyx_t_4 = NULL;
if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) {
__pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3);
if (likely(__pyx_t_4)) {
PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3);
__Pyx_INCREF(__pyx_t_4);
__Pyx_INCREF(function);
__Pyx_DECREF_SET(__pyx_t_3, function);
}
}
__pyx_t_2 = (__pyx_t_4) ? __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_4) : __Pyx_PyObject_CallNoArg(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 162, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
__pyx_r = __pyx_t_2;
__pyx_t_2 = 0;
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
goto __pyx_L0;
}
#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP
__pyx_tp_dict_version = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
__pyx_obj_dict_version = __Pyx_get_object_dict_version(((PyObject *)__pyx_v_self));
if (unlikely(__pyx_type_dict_guard != __pyx_tp_dict_version)) {
__pyx_tp_dict_version = __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
}
#endif
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP
}
#endif
}
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_AddTraceback("rolling_checksum_pyx_mod.Rolling_checksum.get_modulus", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* Python wrapper */
static PyObject *__pyx_pw_24rolling_checksum_pyx_mod_16Rolling_checksum_5get_modulus(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/
static char __pyx_doc_24rolling_checksum_pyx_mod_16Rolling_checksum_4get_modulus[] = "Return the modulus.";
static PyObject *__pyx_pw_24rolling_checksum_pyx_mod_16Rolling_checksum_5get_modulus(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) {
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("get_modulus (wrapper)", 0);
__pyx_r = __pyx_pf_24rolling_checksum_pyx_mod_16Rolling_checksum_4get_modulus(((struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *)__pyx_v_self));
/* function exit code */
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_24rolling_checksum_pyx_mod_16Rolling_checksum_4get_modulus(struct __pyx_obj_24rolling_checksum_pyx_mod_Rolling_checksum *__pyx_v_self) {
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("get_modulus", 0);
__Pyx_XDECREF(__pyx_r);
__pyx_t_1 = __pyx_f_24rolling_checksum_pyx_mod_16Rolling_checksum_get_modulus(__pyx_v_self, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 162, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_r = __pyx_t_1;
__pyx_t_1 = 0;
goto __pyx_L0;
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_AddTraceback("rolling_checksum_pyx_mod.Rolling_checksum.get_modulus", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
163: """Return the modulus."""
+164: return self._modulus
__Pyx_XDECREF(__pyx_r); __pyx_t_1 = __Pyx_PyInt_From_uint64_t(__pyx_v_self->_modulus); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0;