def bitcount(x):
    count = 0
    while x:
        if x & 1:
            count += 1
        x = x >> 1
    return count

class PartOfSpeechModel:

    def __init__(self, lookback=5):
        self.__states = {}
        self.__lookback = lookback

    def add_sentence(self, sentence):

        for i1 in xrange(1, len(sentence)):
            i0 = max(i1 - self.__lookback, 0)
            phrase = sentence[i0:i1]
            next = sentence[i1]

            state = self.__states.setdefault(phrase, {})
            state[next] = state.get(next, 0) + 1

    def add_text(self, txt):
        sentence = []
        for i in xrange(txt.length()):
            t = txt.get_token(i)
            if t.is_stop():
                if not 0 in sentence:
                    print reduce(lambda x, y: x*y, map(bitcount, sentence))
                sentence = []
            elif not t.is_start():
                sentence.append(t.pos_mask())