Detokenizing the words about bert-ner HOT 1 CLOSED

    def join_tokens(tokens):
        res = ''
        if tokens:
            res = tokens[0]
            for token in tokens[1:]:
                if not (token.isalpha() and res[-1].isalpha()):
                    res += token  # punctuation
                else:
                    res += ' ' + token  # regular word
        return res
    
    def collapse(ner_result):
        # List with the result
        collapsed_result = []
    
    
        current_entity_tokens = []
        current_entity = None
    
        # Iterate over the tagged tokens
        for token, tag in ner_result:
    
            if tag.startswith("B-"):
                # ... if we have a previous entity in the buffer, store it in the result list
                if current_entity is not None:
                    collapsed_result.append([join_tokens(current_entity_tokens), current_entity])
    
                current_entity = tag[2:]
                # The new entity has so far only one token
                current_entity_tokens = [token]
    
            # If the entity continues ...
            elif current_entity_tokens!= None and tag == "I-" + str(current_entity):
                # Just add the token buffer
                current_entity_tokens.append(token)
            else:
                collapsed_result.append([join_tokens(current_entity_tokens), current_entity])
                collapsed_result.append([token,tag[2:]])
    
                current_entity_tokens = []
                current_entity = None
    
                pass
    
        # The last entity is still in the buffer, so add it to the result
        # ... but only if there were some entity at all
        if current_entity is not None:
            collapsed_result.append([join_tokens(current_entity_tokens), current_entity])
            collapsed_result = sorted(collapsed_result)
            collapsed_result = list(k for k, _ in itertools.groupby(collapsed_result))
    
        return collapsed_result

text="U.S. Securities and Exchange Commission"
lut = [(token, ix) for ix, word in enumerate(text.split()) for token in tokenize(w)]