From b40c6010f22847f478c9adad3432969536c6c071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Radim=20=C5=98eh=C5=AF=C5=99ek?= Date: Thu, 8 Oct 2020 12:31:25 +0200 Subject: [PATCH 1/5] refactor phrases --- gensim/models/phrases.py | 947 ++++++++++++++++----------------------- 1 file changed, 398 insertions(+), 549 deletions(-) diff --git a/gensim/models/phrases.py b/gensim/models/phrases.py index 9460619db8..5eb4914831 100644 --- a/gensim/models/phrases.py +++ b/gensim/models/phrases.py @@ -11,7 +11,7 @@ * `Mikolov, et. al: "Distributed Representations of Words and Phrases and their Compositionality" `_ -* `"Normalized (Pointwise) Mutual Information in Colocation Extraction" by Gerlof Bouma +* `"Normalized (Pointwise) Mutual Information in Collocation Extraction" by Gerlof Bouma `_ @@ -21,39 +21,42 @@ >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus - >>> from gensim.models.phrases import Phrases, Phraser + >>> from gensim.models.phrases import Phrases, FrozenPhrases >>> - >>> # Load training data. + >>> # Create training corpus. Must be a sequence of sentences (e.g. an iterable or a generator). >>> sentences = Text8Corpus(datapath('testcorpus.txt')) - >>> # The training corpus must be a sequence (stream, generator) of sentences, - >>> # with each sentence a list of tokens: - >>> print(list(sentences)[0][:10]) + >>> # Each sentence must be a list of string tokens: + >>> first_sentence = next(iter(sentences)) + >>> print(first_sentence[:10]) ['computer', 'human', 'interface', 'computer', 'response', 'survey', 'system', 'time', 'user', 'interface'] >>> - >>> # Train a toy bigram model. - >>> phrases = Phrases(sentences, min_count=1, threshold=1) + >>> # Train a toy phrase model on our training corpus. + >>> phrase_model = Phrases(sentences, delimiter='_', min_count=1, threshold=1) + >>> >>> # Apply the trained phrases model to a new, unseen sentence. - >>> phrases[['trees', 'graph', 'minors']] + >>> new_sentence = ['trees', 'graph', 'minors'] + >>> phrase_model[new_sentence] ['trees_graph', 'minors'] >>> # The toy model considered "trees graph" a single phrase => joined the two - >>> # tokens into a single token, `trees_graph`. + >>> # tokens into a single "phrase" token, using our selected `_` delimiter. + >>> + >>> # Apply the trained model to each sentence of a corpus, using the same [] syntax: + >>> for sent in phrase_model[sentences]: + ... pass >>> >>> # Update the model with two new sentences on the fly. - >>> phrases.add_vocab([["hello", "world"], ["meow"]]) + >>> phrase_model.add_vocab([["hello", "world"], ["meow"]]) >>> >>> # Export the trained model = use less RAM, faster processing. Model updates no longer possible. - >>> bigram = Phraser(phrases) - >>> bigram[['trees', 'graph', 'minors']] # apply the exported model to a sentence + >>> frozen_model = phrase_model.freeze() + >>> # Apply the exported model to a sentence; same results as before: + >>> frozen_model[new_sentence] ['trees_graph', 'minors'] >>> - >>> # Apply the exported model to each sentence of a corpus: - >>> for sent in bigram[sentences]: - ... pass - >>> - >>> # Save / load an exported collocation model. - >>> bigram.save("/tmp/my_bigram_model.pkl") - >>> bigram_reloaded = Phraser.load("/tmp/my_bigram_model.pkl") - >>> bigram_reloaded[['trees', 'graph', 'minors']] # apply the exported model to a sentence + >>> # Save / load models. + >>> frozen_model.save("/tmp/my_phrase_model.pkl") + >>> model_reloaded = Phrases.load("/tmp/my_phrase_model.pkl") + >>> model_reloaded[['trees', 'graph', 'minors']] # apply the reloaded model to a sentence ['trees_graph', 'minors'] """ @@ -62,7 +65,6 @@ import os import logging from collections import defaultdict -import functools import itertools from math import log import pickle @@ -73,6 +75,89 @@ logger = logging.getLogger(__name__) +NEGATIVE_INFINITY = float('-inf') + + +def original_scorer(worda_count, wordb_count, bigram_count, len_vocab, min_count, corpus_word_count): + r"""Bigram scoring function, based on the original `Mikolov, et. al: "Distributed Representations + of Words and Phrases and their Compositionality" `_. + + Parameters + ---------- + worda_count : int + Number of occurrences for first word. + wordb_count : int + Number of occurrences for second word. + bigram_count : int + Number of co-occurrences for phrase "worda_wordb". + len_vocab : int + Size of vocabulary. + min_count: int + Minimum collocation count threshold. + corpus_word_count : int + Not used in this particular scoring technique. + + Returns + ------- + float + Score for given bi-gram, greater than or equal to 0. + + Notes + ----- + Formula: :math:`\frac{(bigram\_count - min\_count) * len\_vocab }{ (worda\_count * wordb\_count)}`. + + """ + denom = worda_count * wordb_count + if denom == 0: + return NEGATIVE_INFINITY + return (bigram_count - min_count) / float(denom) * len_vocab + + +def npmi_scorer(worda_count, wordb_count, bigram_count, len_vocab, min_count, corpus_word_count): + r"""Calculation NPMI score based on `"Normalized (Pointwise) Mutual Information in Colocation Extraction" + by Gerlof Bouma `_. + + Parameters + ---------- + worda_count : int + Number of occurrences for first word. + wordb_count : int + Number of occurrences for second word. + bigram_count : int + Number of co-occurrences for phrase "worda_wordb". + len_vocab : int + Not used. + min_count: int + Ignore all bigrams with total collected count lower than this value. + corpus_word_count : int + Total number of words in the corpus. + + Returns + ------- + float + If bigram_count >= min_count, return the collocation score, in the range -1 to 1. + Otherwise return -inf. + + Notes + ----- + Formula: :math:`\frac{ln(prop(word_a, word_b) / (prop(word_a)*prop(word_b)))}{ -ln(prop(word_a, word_b)}`, + where :math:`prob(word) = \frac{word\_count}{corpus\_word\_count}` + + """ + if bigram_count >= min_count: + corpus_word_count = float(corpus_word_count) + pa = worda_count / corpus_word_count + pb = wordb_count / corpus_word_count + pab = bigram_count / corpus_word_count + try: + return log(pab / (pa * pb)) / -log(pab) + except ValueError: # some of the counts were zero => never a phrase + return NEGATIVE_INFINITY + else: + # Return -infinity to make sure that no phrases will be created + # from bigrams less frequent than min_count. + return NEGATIVE_INFINITY + def _is_single(obj): """Check whether `obj` is a single document or an entire corpus. @@ -84,7 +169,8 @@ def _is_single(obj): Return ------ (bool, object) - (is_single, new) tuple, where `new` yields the same sequence as `obj`. + 2-tuple ``(is_single_document, new_obj)`` tuple, where `new_obj` + yields the same sequence as the original `obj`. Notes ----- @@ -97,217 +183,198 @@ def _is_single(obj): peek = next(obj_iter) obj_iter = itertools.chain([peek], obj_iter) except StopIteration: - # An empty object is a single document + # An empty object is interpreted as a single document (not a corpus). return True, obj if isinstance(peek, str): - # It's a document, return the iterator + # First item is a string => obj is a single document for sure. return True, obj_iter if temp_iter is obj: - # Checking for iterator to the object + # An iterator / generator => interpret input as a corpus. return False, obj_iter - else: - # If the first item isn't a string, assume obj is a corpus - return False, obj + # If the first item isn't a string, assume obj is an iterable corpus. + return False, obj -class SentenceAnalyzer: - """Base util class for :class:`~gensim.models.phrases.Phrases` and :class:`~gensim.models.phrases.Phraser`.""" - def score_item(self, worda, wordb, components, scorer): - """Get bi-gram score statistics. +class _PhrasesTransformation(interfaces.TransformationABC): + """ + Abstract base class for :class:`~gensim.models.phrases.Phrases` and + :class:`~gensim.models.phrases.FrozenPhrases`. - Parameters - ---------- - worda : str - First word of bi-gram. - wordb : str - Second word of bi-gram. - components : generator - Contain all phrases. - scorer : function - Scorer function, as given to :class:`~gensim.models.phrases.Phrases`. - See :func:`~gensim.models.phrases.npmi_scorer` and :func:`~gensim.models.phrases.original_scorer`. + """ + def __init__(self, common_terms): + self.common_terms = frozenset(common_terms) + + def score_candidate(self, word_a, word_b, in_between): + """Score a single phrase candidate. Returns ------- - float - Score for given bi-gram. If bi-gram not present in dictionary - return -1. - + (str, float) + 2-tuple of ``(delimiter-joined phrase, phrase score)`` for a phrase, + or ``(None, None)`` if not a phrase. """ - vocab = self.vocab - if worda in vocab and wordb in vocab: - bigram = self.delimiter.join(components) - if bigram in vocab: - return scorer( - worda_count=float(vocab[worda]), - wordb_count=float(vocab[wordb]), - bigram_count=float(vocab[bigram])) - return -1 - - def analyze_sentence(self, sentence, threshold, common_terms, scorer): - """Analyze a sentence, detecting any bigrams that should be concatenated. + raise NotImplementedError("ABC: override this method in child classes") + + def _analyze_sentence(self, sentence): + """Analyze a sentence, concatenating any detected phrases into a single token. Parameters ---------- sentence : iterable of str Token sequence representing the sentence to be analyzed. - threshold : float - The minimum score for a bigram to be taken into account. - common_terms : list of object - List of common terms, they receive special treatment. - scorer : function - Scorer function, as given to :class:`~gensim.models.phrases.Phrases`. - See :func:`~gensim.models.phrases.npmi_scorer` and :func:`~gensim.models.phrases.original_scorer`. Yields ------ (str, score) - If bi-gram detected, a tuple where the first element is a detect bigram, second its score. - Otherwise, the first tuple element is a single word and second is None. + Iterate through the input sentence tokens and yield 2-tuples of: + - ``(concatenated_phrase_tokens, score)`` for token sequences that form a phrase. + - ``(word, None)`` if the token is not a part of a phrase. """ - s = [utils.any2utf8(w) for w in sentence] - # adding None is a trick that helps getting an automatic happy ending - # as it won't be a common_word, nor score - s.append(None) - last_uncommon = None - in_between = [] - for word in s: - is_common = word in common_terms - if not is_common and last_uncommon: - chain = [last_uncommon] + in_between + [word] - # test between last_uncommon - score = self.score_item( - worda=last_uncommon, - wordb=word, - components=chain, - scorer=scorer, - ) - if score > threshold: - yield (chain, score) - last_uncommon = None - in_between = [] + start_token, in_between = None, [] + for word in sentence: + if word not in self.common_terms: + # The current word is a normal token, not a stop word, which means it's a potential + # beginning (or end) of a phrase. + if start_token: + # We're inside a potential phrase, of which this word is the end. + phrase, score = self.score_candidate(start_token, word, in_between) + if score is not None: + # Phrase detected! + yield phrase, score + start_token, in_between = None, [] + else: + # Not a phrase after all. Dissolve the candidate's constituent tokens as individual words. + yield start_token, None + for w in in_between: + yield w, None + start_token, in_between = word, [] # new potential phrase starts here else: - # release words individually - for w in itertools.chain([last_uncommon], in_between): - yield (w, None) - in_between = [] - last_uncommon = word - elif not is_common: - last_uncommon = word - else: # common term - if last_uncommon: - # wait for uncommon resolution + # Not inside a potential bigram yet; start a new potential bigram here. + start_token, in_between = word, [] + else: # We're a stop word. + if start_token: + # We're inside a potential bigram: add the stopword and keep growing the phrase. in_between.append(word) else: - yield (word, None) + # Not inside a bigram: emit the stopword and move on. Phrases never begin with a stopword. + yield word, None + # Emit any non-phrase tokens at the end. + if start_token: + yield start_token, None + for w in in_between: + yield w, None + + def __getitem__(self, sentence): + """Convert the input sequence of tokens `sentence` into a sequence of tokens where adjacent + tokens are replaced by a single token if they form a bigram collocation. + + If `sentence` is an entire corpus (iterable of sentences rather than a single + sentence), return an iterable that converts each of the corpus' sentences + into phrases on the fly, one after another. + + Parameters + ---------- + sentence : {list of str, iterable of list of str} + Input sentence or a stream of sentences. + + Return + ------ + {list of str, iterable of list of str} + Sentence with phrase tokens joined by `self.delimiter` character, if input was a single sentence. + A generator of such joined sentences if input was a corpus. + """ + is_single, sentence = _is_single(sentence) + if not is_single: + # If the input is an entire corpus (rather than a single sentence), + # return an iterable stream. + return self._apply(sentence) -class PhrasesTransformation(interfaces.TransformationABC): - """Base util class for :class:`~gensim.models.phrases.Phrases` and :class:`~gensim.models.phrases.Phraser`.""" + return [token for token, _ in self._analyze_sentence(sentence)] @classmethod def load(cls, *args, **kwargs): """Load a previously saved :class:`~gensim.models.phrases.Phrases` / - :class:`~gensim.models.phrases.Phraser` class. Handles backwards compatibility from older - :class:`~gensim.models.phrases.Phrases` / :class:`~gensim.models.phrases.Phraser` - versions which did not support pluggable scoring functions. + :class:`~gensim.models.phrases.FrozenPhrases` model. + + Handles backwards compatibility from older versions which did not support pluggable scoring functions. Parameters ---------- args : object - Sequence of arguments, see :class:`~gensim.utils.SaveLoad.load` for more information. + See :class:`~gensim.utils.SaveLoad.load`. kwargs : object - Sequence of arguments, see :class:`~gensim.utils.SaveLoad.load` for more information. + See :class:`~gensim.utils.SaveLoad.load`. """ - model = super(PhrasesTransformation, cls).load(*args, **kwargs) - # update older models - # if value in phrasegrams dict is a tuple, load only the scores. + model = super(_PhrasesTransformation, cls).load(*args, **kwargs) - for component, score in getattr(model, "phrasegrams", {}).items(): + # Upgrade FrozenPhrases + try: + phrasegrams = getattr(model, "phrasegrams", {}) + component, score = next(iter(phrasegrams.items())) if isinstance(score, tuple): - frequency, score_val = score - model.phrasegrams[component] = score_val - - # if no scoring parameter, use default scoring + # Value in phrasegrams used to be a tuple; keep only the 2nd tuple component = score. + model.phrasegrams = {model.delimiter.join(key): val[1] for key, val in phrasegrams.items()} + elif isinstance(component, tuple): # 3.8 => 4.0: phrasegram keys are strings, not tuples + model.phrasegrams[model.delimiter.join(component)] = score + except StopIteration: + # no phrasegrams, nothing to upgrade + pass + + # If no scoring parameter, use default scoring. if not hasattr(model, 'scoring'): - logger.info('older version of %s loaded without scoring function', cls.__name__) - logger.info('setting pluggable scoring method to original_scorer for compatibility') + logger.warning('older version of %s loaded without scoring function', cls.__name__) + logger.warning('setting pluggable scoring method to original_scorer for compatibility') model.scoring = original_scorer - # if there is a scoring parameter, and it's a text value, load the proper scoring function + # If there is a scoring parameter, and it's a text value, load the proper scoring function. if hasattr(model, 'scoring'): if isinstance(model.scoring, str): if model.scoring == 'default': - logger.info('older version of %s loaded with "default" scoring parameter', cls.__name__) - logger.info('setting scoring method to original_scorer pluggable scoring method for compatibility') + logger.warning('older version of %s loaded with "default" scoring parameter', cls.__name__) + logger.warning('setting scoring method to original_scorer for compatibility') model.scoring = original_scorer elif model.scoring == 'npmi': - logger.info('older version of %s loaded with "npmi" scoring parameter', cls.__name__) - logger.info('setting scoring method to npmi_scorer pluggable scoring method for compatibility') + logger.warning('older version of %s loaded with "npmi" scoring parameter', cls.__name__) + logger.warning('setting scoring method to npmi_scorer for compatibility') model.scoring = npmi_scorer else: - raise ValueError( - 'failed to load %s model with unknown scoring setting %s' % (cls.__name__, model.scoring)) - # if there is no common_terms attribute, initialize + raise ValueError(f'failed to load {cls.__name__} model, unknown scoring "{model.scoring}"') + # Initialize new attributes to default values. if not hasattr(model, "common_terms"): - logger.info('older version of %s loaded without common_terms attribute', cls.__name__) - logger.info('setting common_terms to empty set') + logger.warning( + 'older version of %s loaded without common_terms attribute, setting it to empty set', + cls.__name__, + ) model.common_terms = frozenset() - return model - - -def _sentence2token(phrase_class, sentence): - """ Convert the input tokens `sentence` into tokens where detected bigrams are joined by a selected delimiter. - - This function is used by: meth:`~gensim.models.phrases.Phrases.__getitem__` and - meth:`~gensim.models.phrases.Phraser.__getitem__` - Parameters - ---------- - phrase_class : - class:`~gensim.models.phrases.Phrases` or :class:`~gensim.models.phrases.Phraser` - sentence : {list of str, iterable of list of str} - Sentence or text corpus. - - Returns - ------- - {list of str, :class:`~gensim.interfaces.TransformedCorpus`} - `sentence` with detected phrase bigrams merged together, or a streamed corpus of such sentences - if the input was a corpus. - - """ - is_single, sentence = _is_single(sentence) - if not is_single: - # if the input is an entire corpus (rather than a single sentence), - # return an iterable stream. - return phrase_class._apply(sentence) - - delimiter = phrase_class.delimiter - if hasattr(phrase_class, 'vocab'): - scorer = functools.partial( - phrase_class.scoring, - len_vocab=float(len(phrase_class.vocab)), - min_count=float(phrase_class.min_count), - corpus_word_count=float(phrase_class.corpus_word_count)) - else: - scorer = None - bigrams = phrase_class.analyze_sentence(sentence, threshold=phrase_class.threshold, - common_terms=phrase_class.common_terms, scorer=scorer) + if not hasattr(model, 'corpus_word_count'): + logger.warning('older version of %s loaded without corpus_word_count', cls.__name__) + logger.warning('setting corpus_word_count to 0, do not use it in your scoring function') + model.corpus_word_count = 0 - new_s = [] - for words, score in bigrams: - if score is not None: - words = delimiter.join(words) - new_s.append(words) - return [utils.to_unicode(w) for w in new_s] + # Before 4.0.0, we stored strings as UTF8 bytes internally, to save RAM. Since 4.0.0, we use strings. + if getattr(model, 'vocab', None): + word = next(iter(model.vocab)) # get a random key – any key will do + if not isinstance(word, str): + logger.info("old version of %s loaded, upgrading %i words in memory", cls.__name__, len(model.vocab)) + logger.info("re-save the loaded model to avoid this upgrade in the future") + model.vocab = { # needs lots of extra RAM temporarily! + str(key, encoding='utf8'): value + for key, value in model.vocab.items() + } + if not isinstance(model.delimiter, str): + model.delimiter = str(model.delimiter, encoding='utf8') + return model -class Phrases(SentenceAnalyzer, PhrasesTransformation): +class Phrases(_PhrasesTransformation): """Detect phrases based on collocation counts.""" def __init__( self, sentences=None, min_count=5, threshold=10.0, - max_vocab_size=40000000, delimiter=b'_', progress_per=10000, + max_vocab_size=40000000, delimiter='_', progress_per=10000, scoring='default', common_terms=frozenset(), ): """ @@ -330,7 +397,7 @@ def __init__( to keep memory under control. The default of 40M needs about 3.6GB of RAM. Increase/decrease `max_vocab_size` depending on how much available memory you have. delimiter : str, optional - Glue character used to join collocation tokens, should be a byte string (e.g. b'_'). + Glue character used to join collocation tokens. scoring : {'default', 'npmi', function}, optional Specify how potential phrases are scored. `scoring` can be set with either a string that refers to a built-in scoring function, or with a function with the expected parameter names. @@ -359,9 +426,40 @@ def __init__( * corpus_word_count - the total number of tokens (non-unique) in `sentences` The scoring function **must accept all these parameters**, even if it doesn't use them in its scoring. + The scoring function **must be pickleable**. + Examples + ---------- + .. sourcecode:: pycon + + >>> from gensim.test.utils import datapath + >>> from gensim.models.word2vec import Text8Corpus + >>> from gensim.models.phrases import Phrases, FrozenPhrases + >>> + >>> # Load corpus and train a model. + >>> sentences = Text8Corpus(datapath('testcorpus.txt')) + >>> phrases = Phrases(sentences, min_count=1, threshold=1) + >>> + >>> # Use the model to detect phrases in a new sentence. + >>> sent = [u'trees', u'graph', u'minors'] + >>> print(phrases[sent]) + [u'trees_graph', u'minors'] + >>> + >>> # Or transform multiple sentences at once. + >>> sents = [[u'trees', u'graph', u'minors'], [u'graph', u'minors']] + >>> for phrase in frozen_phrases[sents]: + ... print(phrase) + [u'trees_graph', u'minors'] + [u'graph_minors'] + >>> + >>> # Export a FrozenPhrases object that is more efficient but doesn't allow any more training. + >>> frozen_phrases = phrases.freeze() + >>> print(frozen_phrases[sent]) + [u'trees_graph', u'minors'] + """ + super().__init__(common_terms=common_terms) if min_count <= 0: raise ValueError("min_count should be at least 1") @@ -370,10 +468,9 @@ def __init__( if scoring == 'npmi' and (threshold < -1 or threshold > 1): raise ValueError("threshold should be between -1 and 1 for npmi scoring") - # set scoring based on string - # intentially override the value of the scoring parameter rather than set self.scoring here, - # to still run the check of scoring function parameters in the next code block - + # Set scoring based on string. + # Intentially override the value of the scoring parameter rather than set self.scoring here, + # to still run the check of scoring function parameters in the next code block. if isinstance(scoring, str): if scoring == 'default': scoring = original_scorer @@ -382,65 +479,46 @@ def __init__( else: raise ValueError(f'unknown scoring method string {scoring} specified') - scoring_parameters = [ + scoring_params = [ 'worda_count', 'wordb_count', 'bigram_count', 'len_vocab', 'min_count', 'corpus_word_count', ] if callable(scoring): - if all(parameter in getargspec(scoring)[0] for parameter in scoring_parameters): + missing = [param for param in scoring_params if param not in getargspec(scoring)[0]] + if not missing: self.scoring = scoring else: - raise ValueError('scoring function missing expected parameters') + raise ValueError(f'scoring function missing expected parameters {missing}') self.min_count = min_count self.threshold = threshold self.max_vocab_size = max_vocab_size - self.vocab = defaultdict(int) # mapping between utf8 token => its count + self.vocab = defaultdict(int) # mapping between token => its count self.min_reduce = 1 # ignore any tokens with count smaller than this self.delimiter = delimiter self.progress_per = progress_per self.corpus_word_count = 0 - self.common_terms = frozenset(utils.any2utf8(w) for w in common_terms) - # ensure picklability of custom scorer + # Ensure picklability of the scorer. try: pickle.loads(pickle.dumps(self.scoring)) except pickle.PickleError: - raise pickle.PickleError('Custom Phrases scoring function must be pickle-able') + raise pickle.PickleError(f'Custom scoring function in {self.__class__.__name__} must be pickle-able') if sentences is not None: self.add_vocab(sentences) - @classmethod - def load(cls, *args, **kwargs): - """Load a previously saved Phrases class. - Handles backwards compatibility from older Phrases versions which did not support pluggable scoring functions. - - Parameters - ---------- - args : object - Sequence of arguments, see :class:`~gensim.utils.SaveLoad.load` for more information. - kwargs : object - Sequence of arguments, see :class:`~gensim.utils.SaveLoad.load` for more information. - - """ - model = super(Phrases, cls).load(*args, **kwargs) - if not hasattr(model, 'corpus_word_count'): - logger.info('older version of %s loaded without corpus_word_count', cls.__name__) - logger.info('Setting it to 0, do not use it in your scoring function.') - model.corpus_word_count = 0 - return model - def __str__(self): - """Get short string representation of this phrase detector.""" + """Get a short string representation of this phrase detector.""" return "%s<%i vocab, min_count=%s, threshold=%s, max_vocab_size=%s>" % ( self.__class__.__name__, len(self.vocab), self.min_count, self.threshold, self.max_vocab_size, ) @staticmethod - def learn_vocab(sentences, max_vocab_size, delimiter=b'_', progress_per=10000, - common_terms=frozenset()): - """Collect unigram/bigram counts from the `sentences` iterable. + def _learn_vocab( + sentences, max_vocab_size, delimiter='_', common_terms=frozenset(), progress_per=10000, + ): + """Collect unigram and bigram counts from the `sentences` iterable. Parameters ---------- @@ -451,62 +529,40 @@ def learn_vocab(sentences, max_vocab_size, delimiter=b'_', progress_per=10000, for such examples. max_vocab_size : int Maximum size (number of tokens) of the vocabulary. Used to control pruning of less common words, - to keep memory under control. The default of 40M needs about 3.6GB of RAM. Increase/decrease + to keep memory under control. 40M needs about 3.6GB of RAM. Increase/decrease `max_vocab_size` depending on how much available memory you have. delimiter : str, optional - Glue character used to join collocation tokens, should be a byte string (e.g. b'_'). - progress_per : int - Write logs every `progress_per` sentence. + Glue character used to join collocation tokens. common_terms : set of str, optional - List of "stop words" that won't affect frequency count of expressions containing them. - Allow to detect expressions like "bank_of_america" or "eye_of_the_beholder". + List of "stop words" that won't affect frequency count of phrases containing them. + Allow to detect phrases like "bank_of_america" or "eye_of_the_beholder". + progress_per : int + Log progress once every `progress_per` sentences. Return ------ (int, dict of (str, int), int) - Number of pruned words, counters for each word/bi-gram and total number of words. - - Example - ---------- - .. sourcecode:: pycon - - >>> from gensim.test.utils import datapath - >>> from gensim.models.word2vec import Text8Corpus - >>> from gensim.models.phrases import Phrases - >>> - >>> sentences = Text8Corpus(datapath('testcorpus.txt')) - >>> pruned_words, counters, total_words = Phrases.learn_vocab(sentences, 100) - >>> (pruned_words, total_words) - (1, 29) - >>> counters['computer'] - 2 - >>> counters['response_time'] - 1 + Number of pruned words, counters for each word/bi-gram, and total number of words. """ - sentence_no = -1 - total_words = 0 - logger.info("collecting all words and their counts") + sentence_no, total_words, min_reduce = -1, 0, 1 vocab = defaultdict(int) - min_reduce = 1 + logger.info("collecting all words and their counts") for sentence_no, sentence in enumerate(sentences): if sentence_no % progress_per == 0: logger.info( "PROGRESS: at sentence #%i, processed %i words and %i word types", sentence_no, total_words, len(vocab), ) - s = [utils.any2utf8(w) for w in sentence] - last_uncommon = None - in_between = [] - for word in s: + start_token, in_between = None, [] + for word in sentence: if word not in common_terms: vocab[word] += 1 - if last_uncommon is not None: - components = itertools.chain([last_uncommon], in_between, [word]) - vocab[delimiter.join(components)] += 1 - last_uncommon = word - in_between = [] - elif last_uncommon is not None: + if start_token is not None: + phrase_tokens = itertools.chain([start_token], in_between, [word]) + vocab[delimiter.join(phrase_tokens)] += 1 + start_token, in_between = word, [] # treat word as both end of a phrase AND beginning of another + elif start_token is not None: in_between.append(word) total_words += 1 @@ -515,13 +571,13 @@ def learn_vocab(sentences, max_vocab_size, delimiter=b'_', progress_per=10000, min_reduce += 1 logger.info( - "collected %i word types from a corpus of %i words (unigram + bigrams) and %i sentences", - len(vocab), total_words, sentence_no + 1 + "collected %i token types (unigram + bigrams) from a corpus of %i words and %i sentences", + len(vocab), total_words, sentence_no + 1, ) return min_reduce, vocab, total_words def add_vocab(self, sentences): - """Update model with new `sentences`. + """Update model parameters with new `sentences`. Parameters ---------- @@ -554,11 +610,13 @@ def add_vocab(self, sentences): # directly, but gives the new sentences a fighting chance to collect # sufficient counts, before being pruned out by the (large) accumulated # counts collected in previous learn_vocab runs. - min_reduce, vocab, total_words = self.learn_vocab( - sentences, self.max_vocab_size, self.delimiter, self.progress_per, self.common_terms) + min_reduce, vocab, total_words = self._learn_vocab( + sentences, max_vocab_size=self.max_vocab_size, delimiter=self.delimiter, + progress_per=self.progress_per, common_terms=self.common_terms, + ) self.corpus_word_count += total_words - if len(self.vocab) > 0: + if self.vocab: logger.info("merging %i counts into %s", len(vocab), self) self.min_reduce = max(self.min_reduce, min_reduce) for word, count in vocab.items(): @@ -566,225 +624,59 @@ def add_vocab(self, sentences): if len(self.vocab) > self.max_vocab_size: utils.prune_vocab(self.vocab, self.min_reduce) self.min_reduce += 1 - logger.info("merged %s", self) else: - # in common case, avoid doubling gigantic dict - logger.info("using %i counts as vocab in %s", len(vocab), self) + # Optimization for a common case: the current vocab is empty, so apply + # the new vocab directly, no need to double it in memory. self.vocab = vocab - - def export_phrases(self, sentences, out_delimiter=b' ', as_tuples=False): - """Get all phrases that appear in 'sentences' that pass the bigram threshold. - - Parameters - ---------- - sentences : iterable of list of str - Text corpus. - out_delimiter : str, optional - Delimiter used to "glue" together words that form a bigram phrase. - as_tuples : bool, optional - Yield `(tuple(words), score)` instead of `(out_delimiter.join(words), score)`? - - Yields - ------ - ((str, str), float) **or** (str, float) - Phrases detected in `sentences`. Return type depends on the `as_tuples` parameter. - - Example - ------- - .. sourcecode:: pycon - - >>> from gensim.test.utils import datapath - >>> from gensim.models.word2vec import Text8Corpus - >>> from gensim.models.phrases import Phrases - >>> - >>> sentences = Text8Corpus(datapath('testcorpus.txt')) - >>> phrases = Phrases(sentences, min_count=1, threshold=0.1) - >>> - >>> for phrase, score in phrases.export_phrases(sentences): - ... pass - - """ - analyze_sentence = functools.partial( - self.analyze_sentence, - threshold=self.threshold, - common_terms=self.common_terms, - scorer=functools.partial( - self.scoring, - len_vocab=float(len(self.vocab)), - min_count=float(self.min_count), - corpus_word_count=float(self.corpus_word_count), - ), + logger.info("merged %s", self) + + def score_candidate(self, word_a, word_b, in_between): + # Micro optimization: check for quick early-out conditions, before the actual scoring. + word_a_cnt = self.vocab[word_a] + if word_a_cnt <= 0: + return None, None + + word_b_cnt = self.vocab[word_b] + if word_b_cnt <= 0: + return None, None + + phrase = self.delimiter.join([word_a] + in_between + [word_b]) + # XXX: Why do we care about *all* phrase tokens? Why not just score the start+end bigram? + phrase_cnt = self.vocab[phrase] + if phrase_cnt <= 0: + return None, None + + score = self.scoring( + worda_count=word_a_cnt, wordb_count=word_b_cnt, bigram_count=phrase_cnt, + len_vocab=len(self.vocab), min_count=self.min_count, corpus_word_count=self.corpus_word_count, ) - for sentence in sentences: - bigrams = analyze_sentence(sentence) - # keeps only not None scores - filtered = ((words, score) for words, score in bigrams if score is not None) - for words, score in filtered: - if as_tuples: - yield (tuple(words), score) - else: - yield (out_delimiter.join(words), score) + if score <= self.threshold: + return None, None - def __getitem__(self, sentence): - """Convert the input tokens `sentence` into tokens where detected bigrams are joined by a selected delimiter. + return phrase, score - If `sentence` is an entire corpus (iterable of sentences rather than a single - sentence), return an iterable that converts each of the corpus' sentences - into phrases on the fly, one after another. + def freeze(self): + """ + Return an object that contains the bare minimum of information while still allowing + phrase detection. See :class:`~gensim.models.phrases.FrozenPhrases`. - Parameters - ---------- - sentence : {list of str, iterable of list of str} - Sentence or text corpus. + Use this "frozen model" to dramatically reduce RAM footprint if you don't plan to + make any further changes to your `Phrases` model. Returns ------- - {list of str, :class:`gensim.interfaces.TransformedCorpus`} - `sentence` with detected phrase bigrams merged together, or a streamed corpus of such sentences - if the input was a corpus. - - Examples - ---------- - .. sourcecode:: pycon - - >>> from gensim.test.utils import datapath - >>> from gensim.models.word2vec import Text8Corpus - >>> from gensim.models.phrases import Phrases, Phraser - >>> - >>> # Create corpus - >>> sentences = Text8Corpus(datapath('testcorpus.txt')) - >>> - >>> # Train the detector with: - >>> phrases = Phrases(sentences, min_count=1, threshold=1) - >>> # Input is a list of unicode strings: - >>> sent = [u'trees', u'graph', u'minors'] - >>> # Both of these tokens appear in corpus at least twice, and phrase score is higher, than treshold = 1: - >>> print(phrases[sent]) - [u'trees_graph', u'minors'] - >>> - >>> sentences = Text8Corpus(datapath('testcorpus.txt')) - >>> phrases = Phrases(sentences, min_count=1, threshold=1) - >>> phraser = Phraser(phrases) # for speedup - >>> - >>> sent = [[u'trees', u'graph', u'minors'], [u'graph', u'minors']] - >>> for phrase in phraser[sent]: - ... pass + :class:`~gensim.models.phrases.FrozenPhrases` + Exported object that's smaller, faster, but doesn't support model updates. """ - return _sentence2token(self, sentence) - - -def original_scorer(worda_count, wordb_count, bigram_count, len_vocab, min_count, corpus_word_count): - r"""Bigram scoring function, based on the original `Mikolov, et. al: "Distributed Representations - of Words and Phrases and their Compositionality" `_. - - Parameters - ---------- - worda_count : int - Number of occurrences for first word. - wordb_count : int - Number of occurrences for second word. - bigram_count : int - Number of co-occurrences for phrase "worda_wordb". - len_vocab : int - Size of vocabulary. - min_count: int - Minimum collocation count threshold. - corpus_word_count : int - Not used in this particular scoring technique. - - Returns - ------- - float - Score for given bi-gram, greater than or equal to 0. - - Notes - ----- - Formula: :math:`\frac{(bigram\_count - min\_count) * len\_vocab }{ (worda\_count * wordb\_count)}`. - - """ - return (bigram_count - min_count) / worda_count / wordb_count * len_vocab - - -def npmi_scorer(worda_count, wordb_count, bigram_count, len_vocab, min_count, corpus_word_count): - r"""Calculation NPMI score based on `"Normalized (Pointwise) Mutual Information in Colocation Extraction" - by Gerlof Bouma `_. - - Parameters - ---------- - worda_count : int - Number of occurrences for first word. - wordb_count : int - Number of occurrences for second word. - bigram_count : int - Number of co-occurrences for phrase "worda_wordb". - len_vocab : int - Not used. - min_count: int - Ignore all bigrams with total collected count lower than this value. - corpus_word_count : int - Total number of words in the corpus. - - Returns - ------- - float - Score for given bi-gram, in the range -1 to 1. - - Notes - ----- - Formula: :math:`\frac{ln(prop(word_a, word_b) / (prop(word_a)*prop(word_b)))}{ -ln(prop(word_a, word_b)}`, - where :math:`prob(word) = \frac{word\_count}{corpus\_word\_count}` + return FrozenPhrases(self) - """ - if bigram_count >= min_count: - pa = worda_count / corpus_word_count - pb = wordb_count / corpus_word_count - pab = bigram_count / corpus_word_count - return log(pab / (pa * pb)) / -log(pab) - else: - # Return -infinity to make sure that no phrases will be created - # from bigrams less frequent than min_count - return float('-inf') - - -def pseudocorpus(source_vocab, sep, common_terms=frozenset()): - """Feeds `source_vocab`'s compound keys back to it, to discover phrases. - - Parameters - ---------- - source_vocab : iterable of list of str - Tokens vocabulary. - sep : str - Separator element. - common_terms : set, optional - Immutable set of stopwords. - - Yields - ------ - list of str - Phrase. - """ - for k in source_vocab: - if sep not in k: - continue - unigrams = k.split(sep) - for i in range(1, len(unigrams)): - if unigrams[i - 1] not in common_terms: - # do not join common terms - cterms = list(itertools.takewhile(lambda w: w in common_terms, unigrams[i:])) - tail = unigrams[i + len(cterms):] - components = [sep.join(unigrams[:i])] + cterms - if tail: - components.append(sep.join(tail)) - yield components - - -class Phraser(SentenceAnalyzer, PhrasesTransformation): +class FrozenPhrases(_PhrasesTransformation): """Minimal state & functionality exported from :class:`~gensim.models.phrases.Phrases`. The goal of this class is to cut down memory consumption of `Phrases`, by discarding model state - not strictly needed for the bigram detection task. + not strictly needed for the phrase detection task. Use this instead of `Phrases` if you do not need to update the bigram statistics with new documents any more. @@ -796,27 +688,40 @@ def __init__(self, phrases_model): Parameters ---------- phrases_model : :class:`~gensim.models.phrases.Phrases` - Trained phrases instance. + Trained phrases instance, to extract all phrases from. Notes ----- - After the one-time initialization, a :class:`~gensim.models.phrases.Phraser` will be much smaller and somewhat + After the one-time initialization, a :class:`~gensim.models.phrases.FrozenPhrases` will be much smaller and somewhat faster than using the full :class:`~gensim.models.phrases.Phrases` model. Examples - -------- + ---------- .. sourcecode:: pycon >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus - >>> from gensim.models.phrases import Phrases, Phraser + >>> from gensim.models.phrases import Phrases, FrozenPhrases >>> + >>> # Load corpus and train a model. >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> phrases = Phrases(sentences, min_count=1, threshold=1) >>> - >>> bigram = Phraser(phrases) + >>> # Use the model to detect phrases in a new sentence. >>> sent = [u'trees', u'graph', u'minors'] - >>> print(bigram[sent]) + >>> print(phrases[sent]) + [u'trees_graph', u'minors'] + >>> + >>> # Or transform multiple sentences at once. + >>> sents = [[u'trees', u'graph', u'minors'], [u'graph', u'minors']] + >>> for phrase in frozen_phrases[sents]: + ... print(phrase) + [u'trees_graph', u'minors'] + [u'graph_minors'] + >>> + >>> # Export a FrozenPhrases object that is more efficient but doesn't allow any more training. + >>> frozen_phrases = phrases.freeze() + >>> print(frozen_phrases[sent]) [u'trees_graph', u'minors'] """ @@ -825,99 +730,44 @@ def __init__(self, phrases_model): self.delimiter = phrases_model.delimiter self.scoring = phrases_model.scoring self.common_terms = phrases_model.common_terms - corpus = self.pseudocorpus(phrases_model) - self.phrasegrams = {} - logger.info('source_vocab length %i', len(phrases_model.vocab)) - count = 0 - for bigram, score in phrases_model.export_phrases(corpus, self.delimiter, as_tuples=True): - if bigram in self.phrasegrams: - logger.info('Phraser repeat %s', bigram) - self.phrasegrams[bigram] = score - count += 1 - if not count % 50000: - logger.info('Phraser added %i phrasegrams', count) - logger.info('Phraser built with %i phrasegrams', len(self.phrasegrams)) - - def pseudocorpus(self, phrases_model): - """Alias for :func:`gensim.models.phrases.pseudocorpus`. + logger.info('exporting phrases from %s', phrases_model) + self.phrasegrams = self._export_phrases(phrases_model) + logger.info('exported %s', self) - Parameters - ---------- - phrases_model : :class:`~gensim.models.phrases.Phrases` - Phrases instance. - - Return - ------ - generator - Generator with phrases. - - """ - return pseudocorpus(phrases_model.vocab, phrases_model.delimiter, phrases_model.common_terms) - - def score_item(self, worda, wordb, components, scorer): - """Score a bigram. + def __str__(self): + """Get a short string representation of this phrase detector.""" + return "%s<%i phrases, min_count=%s, threshold=%s>" % ( + self.__class__.__name__, len(self.phrasegrams), self.min_count, self.threshold, + ) - Parameters - ---------- - worda : str - First word for comparison. - wordb : str - Second word for comparison. - components : generator - Contain phrases. - scorer : {'default', 'npmi'} - NOT USED. + def _export_phrases(self, phrases_model): + """Extract all phrases that pass the threshold out of `phrases_model`. Returns - ------- - float - Score for given bi-gram, if bi-gram not presented in dictionary - return -1. + ------ + dict[str, float] + Mapping between phrases and their scores. """ - try: - return self.phrasegrams[tuple(components)] - except KeyError: - return -1 + result, source_vocab = {}, phrases_model.vocab + for token in source_vocab: + unigrams = token.split(self.delimiter) + if len(unigrams) < 2: + continue # no phrases here + phrase, score = phrases_model.score_candidate(unigrams[0], unigrams[-1], unigrams[1:-1]) + if score is not None: + result[phrase] = score + return result - def __getitem__(self, sentence): - """Convert the input sequence of tokens `sentence` into a sequence of tokens where adjacent - tokens are replaced by a single token if they form a bigram collocation. + def score_candidate(self, word_a, word_b, in_between): + phrase = self.delimiter.join([word_a] + in_between + [word_b]) + score = self.phrasegrams.get(phrase, NEGATIVE_INFINITY) + if score > self.threshold: + return phrase, score + return None, None - Parameters - ---------- - sentence : {list of str, iterable of list of str} - Input sentence or a stream of sentences. - - Return - ------ - {list of str, iterable of list of str} - Sentence or sentences with phrase tokens joined by `self.delimiter` character. - Examples - ---------- - .. sourcecode:: pycon - - >>> from gensim.test.utils import datapath - >>> from gensim.models.word2vec import Text8Corpus - >>> from gensim.models.phrases import Phrases, Phraser - >>> - >>> sentences = Text8Corpus(datapath('testcorpus.txt')) # Read corpus - >>> - >>> phrases = Phrases(sentences, min_count=1, threshold=1) # Train model - >>> # Create a Phraser object to transform any sentence and turn 2 suitable tokens into 1 phrase - >>> phraser_model = Phraser(phrases) - >>> - >>> sent = [u'trees', u'graph', u'minors'] - >>> print(phraser_model[sent]) - [u'trees_graph', u'minors'] - >>> sent = [[u'trees', u'graph', u'minors'], [u'graph', u'minors']] - >>> for phrase in phraser_model[sent]: - ... print(phrase) - [u'trees_graph', u'minors'] - [u'graph_minors'] - - """ - return _sentence2token(self, sentence) +Phraser = FrozenPhrases # alias for backward compatibility if __name__ == '__main__': @@ -935,7 +785,6 @@ def __getitem__(self, sentence): from gensim.models.word2vec import Text8Corpus sentences = Text8Corpus(infile) - # test_doc = LineSentence('test/test_data/testcorpus.txt') bigram = Phrases(sentences, min_count=5, threshold=100) for s in bigram[sentences]: - print(utils.to_utf8(u' '.join(s))) + print(u' '.join(s)) From 6f4053bdd602a29ccf0532b18d6f8e80e3352a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Radim=20=C5=98eh=C5=AF=C5=99ek?= Date: Sat, 10 Oct 2020 11:02:46 +0200 Subject: [PATCH 2/5] fix phrases tests --- gensim/models/phrases.py | 70 ++++-- gensim/test/test_phrases.py | 458 +++++++++++++----------------------- 2 files changed, 213 insertions(+), 315 deletions(-) diff --git a/gensim/models/phrases.py b/gensim/models/phrases.py index 5eb4914831..43806d36a9 100644 --- a/gensim/models/phrases.py +++ b/gensim/models/phrases.py @@ -215,7 +215,7 @@ def score_candidate(self, word_a, word_b, in_between): """ raise NotImplementedError("ABC: override this method in child classes") - def _analyze_sentence(self, sentence): + def analyze_sentence(self, sentence): """Analyze a sentence, concatenating any detected phrases into a single token. Parameters @@ -291,7 +291,40 @@ def __getitem__(self, sentence): # return an iterable stream. return self._apply(sentence) - return [token for token, _ in self._analyze_sentence(sentence)] + return [token for token, _ in self.analyze_sentence(sentence)] + + def export_phrases(self, sentences): + """Get all unique phrases (multi-word expressions) that appear in 'sentences'. + + Parameters + ---------- + sentences : iterable of list of str + Text corpus. + + Returns + ------- + set((str, float)) + Set of unique phrases and their scores. + + Example + ------- + .. sourcecode:: pycon + >>> from gensim.test.utils import datapath + >>> from gensim.models.word2vec import Text8Corpus + >>> from gensim.models.phrases import Phrases + >>> + >>> sentences = Text8Corpus(datapath('testcorpus.txt')) + >>> phrases = Phrases(sentences, min_count=1, threshold=0.1) + >>> + >>> for phrase, score in phrases.export_phrases(sentences): + ... print(phrase, score) + """ + result = set() + for sentence in sentences: + for phrase, score in self.analyze_sentence(sentence): + if score is not None: + result.add((phrase, score)) + return result @classmethod def load(cls, *args, **kwargs): @@ -316,9 +349,15 @@ def load(cls, *args, **kwargs): component, score = next(iter(phrasegrams.items())) if isinstance(score, tuple): # Value in phrasegrams used to be a tuple; keep only the 2nd tuple component = score. - model.phrasegrams = {model.delimiter.join(key): val[1] for key, val in phrasegrams.items()} - elif isinstance(component, tuple): # 3.8 => 4.0: phrasegram keys are strings, not tuples - model.phrasegrams[model.delimiter.join(component)] = score + model.phrasegrams = { + str(model.delimiter.join(key), encoding='utf8'): val[1] + for key, val in phrasegrams.items() + } + elif isinstance(component, tuple): # 3.8 => 4.0: phrasegram keys are strings, not tuples with bytestrings + model.phrasegrams = { + str(model.delimiter.join(component), encoding='utf8'): score + for key, val in phrasegrams.items() + } except StopIteration: # no phrasegrams, nothing to upgrade pass @@ -360,10 +399,10 @@ def load(cls, *args, **kwargs): if not isinstance(word, str): logger.info("old version of %s loaded, upgrading %i words in memory", cls.__name__, len(model.vocab)) logger.info("re-save the loaded model to avoid this upgrade in the future") - model.vocab = { # needs lots of extra RAM temporarily! - str(key, encoding='utf8'): value - for key, value in model.vocab.items() - } + vocab = defaultdict(int) + for key, value in model.vocab.items(): # needs lots of extra RAM temporarily! + vocab[str(key, encoding='utf8')] = value + model.vocab = vocab if not isinstance(model.delimiter, str): model.delimiter = str(model.delimiter, encoding='utf8') return model @@ -508,7 +547,6 @@ def __init__( self.add_vocab(sentences) def __str__(self): - """Get a short string representation of this phrase detector.""" return "%s<%i vocab, min_count=%s, threshold=%s, max_vocab_size=%s>" % ( self.__class__.__name__, len(self.vocab), self.min_count, self.threshold, self.max_vocab_size, @@ -591,7 +629,8 @@ def add_vocab(self, sentences): >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus >>> from gensim.models.phrases import Phrases - >>> # Create corpus and use it for phrase detector + >>> + >>> # Train a phrase detector from a text corpus. >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> phrases = Phrases(sentences) # train model >>> assert len(phrases.vocab) == 37 @@ -605,8 +644,8 @@ def add_vocab(self, sentences): >>> assert len(phrases.vocab) == 60 """ - # uses a separate vocab to collect the token counts from `sentences`. - # this consumes more RAM than merging new sentences into `self.vocab` + # Uses a separate vocab to collect the token counts from `sentences`. + # This consumes more RAM than merging new sentences into `self.vocab` # directly, but gives the new sentences a fighting chance to collect # sufficient counts, before being pruned out by the (large) accumulated # counts collected in previous learn_vocab runs. @@ -731,16 +770,15 @@ def __init__(self, phrases_model): self.scoring = phrases_model.scoring self.common_terms = phrases_model.common_terms logger.info('exporting phrases from %s', phrases_model) - self.phrasegrams = self._export_phrases(phrases_model) + self.phrasegrams = self._import_phrases(phrases_model) logger.info('exported %s', self) def __str__(self): - """Get a short string representation of this phrase detector.""" return "%s<%i phrases, min_count=%s, threshold=%s>" % ( self.__class__.__name__, len(self.phrasegrams), self.min_count, self.threshold, ) - def _export_phrases(self, phrases_model): + def _import_phrases(self, phrases_model): """Extract all phrases that pass the threshold out of `phrases_model`. Returns diff --git a/gensim/test/test_phrases.py b/gensim/test/test_phrases.py index ed85fea2b5..cec2747073 100644 --- a/gensim/test/test_phrases.py +++ b/gensim/test/test_phrases.py @@ -4,140 +4,97 @@ # Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html """ -Automated tests for checking transformation algorithms (the models package). +Automated tests for the phrase detection module. """ - import logging import unittest -import six import numpy as np -from gensim.utils import to_unicode -from gensim.models.phrases import SentenceAnalyzer, Phrases, Phraser -from gensim.models.phrases import pseudocorpus, original_scorer +from gensim.models.phrases import Phrases, FrozenPhrases, _PhrasesTransformation +from gensim.models.phrases import original_scorer from gensim.test.utils import common_texts, temporary_file, datapath -class TestUtils(unittest.TestCase): - - def test_pseudocorpus_no_common_terms(self): - vocab = [ - "prime_minister", - "gold", - "chief_technical_officer", - "effective"] - result = list(pseudocorpus(vocab, "_")) - self.assertEqual( - result, - [["prime", "minister"], - ["chief", "technical_officer"], - ["chief_technical", "officer"]]) - - def test_pseudocorpus_with_common_terms(self): - vocab = [ - "hall_of_fame", - "gold", - "chief_of_political_bureau", - "effective", - "beware_of_the_dog_in_the_yard"] - common_terms = frozenset(["in", "the", "of"]) - result = list(pseudocorpus(vocab, "_", common_terms=common_terms)) - self.assertEqual( - result, - [["hall", "of", "fame"], - ["chief", "of", "political_bureau"], - ["chief_of_political", "bureau"], - ["beware", "of", "the", "dog_in_the_yard"], - ["beware_of_the_dog", "in", "the", "yard"]]) - - class TestPhraseAnalysis(unittest.TestCase): - class AnalysisTester(SentenceAnalyzer): + class AnalysisTester(_PhrasesTransformation): - def __init__(self, scores): + def __init__(self, scores, threshold): + super().__init__(common_terms={"a", "the", "with", "of"}) self.scores = scores + self.threshold = threshold - def score_item(self, worda, wordb, components, scorer): - """Override for test purpose""" - if worda is not None and wordb is not None: - bigram_word = b"_".join(components) - return self.scores.get(bigram_word, -1) - else: - return -1 - - def analyze(self, scores, sentence): - analyzer = self.AnalysisTester(scores) - return list(analyzer.analyze_sentence( - sentence, - threshold=1, - common_terms={b"a", b"the", b"with", b"of"}, - scorer=None)) - - def analyze_words(self, scores, sentence): - result = ( - w if isinstance(w, (tuple, list)) else [w] - for w, score in self.analyze(scores, sentence)) - return [b"_".join(w).decode("utf-8") for w in result] + def score_candidate(self, word_a, word_b, in_between): + phrase = "_".join([word_a] + in_between + [word_b]) + score = self.scores.get(phrase, -1) + if score > self.threshold: + return phrase, score + return None, None def test_simple_analysis(self): - s = ["simple", "sentence", "should", "pass"] - result = self.analyze_words({}, s) - self.assertEqual(result, s) - s = ["a", "simple", "sentence", "with", "no", "bigram", "but", "common", "terms"] - result = self.analyze_words({}, s) - self.assertEqual(result, s) + """Test transformation with no phrases.""" + sentence = ["simple", "sentence", "should", "pass"] + result = self.AnalysisTester({}, threshold=1)[sentence] + self.assertEqual(result, sentence) + sentence = ["a", "simple", "sentence", "with", "no", "bigram", "but", "common", "terms"] + result = self.AnalysisTester({}, threshold=1)[sentence] + self.assertEqual(result, sentence) def test_analysis_bigrams(self): scores = { - b"simple_sentence": 2, b"sentence_many": 2, - b"many_possible": 2, b"possible_bigrams": 2} - s = ["simple", "sentence", "many", "possible", "bigrams"] - result = self.analyze_words(scores, s) + "simple_sentence": 2, "sentence_many": 2, + "many_possible": 2, "possible_bigrams": 2, + } + sentence = ["simple", "sentence", "many", "possible", "bigrams"] + result = self.AnalysisTester(scores, threshold=1)[sentence] self.assertEqual(result, ["simple_sentence", "many_possible", "bigrams"]) - s = ["some", "simple", "sentence", "many", "bigrams"] - result = self.analyze_words(scores, s) + sentence = ["some", "simple", "sentence", "many", "bigrams"] + result = self.AnalysisTester(scores, threshold=1)[sentence] self.assertEqual(result, ["some", "simple_sentence", "many", "bigrams"]) - s = ["some", "unrelated", "simple", "words"] - result = self.analyze_words(scores, s) - self.assertEqual(result, s) + sentence = ["some", "unrelated", "simple", "words"] + result = self.AnalysisTester(scores, threshold=1)[sentence] + self.assertEqual(result, sentence) def test_analysis_common_terms(self): scores = { - b"simple_sentence": 2, b"sentence_many": 2, - b"many_possible": 2, b"possible_bigrams": 2} - s = ["a", "simple", "sentence", "many", "the", "possible", "bigrams"] - result = self.analyze_words(scores, s) + "simple_sentence": 2, "sentence_many": 2, + "many_possible": 2, "possible_bigrams": 2, + } + sentence = ["a", "simple", "sentence", "many", "the", "possible", "bigrams"] + result = self.AnalysisTester(scores, threshold=1)[sentence] self.assertEqual(result, ["a", "simple_sentence", "many", "the", "possible_bigrams"]) - s = ["simple", "the", "sentence", "and", "many", "possible", "bigrams", "with", "a"] - result = self.analyze_words(scores, s) - self.assertEqual(result, [ - "simple", "the", "sentence", "and", "many_possible", "bigrams", "with", "a"]) + sentence = ["simple", "the", "sentence", "and", "many", "possible", "bigrams", "with", "a"] + result = self.AnalysisTester(scores, threshold=1)[sentence] + self.assertEqual( + result, + ["simple", "the", "sentence", "and", "many_possible", "bigrams", "with", "a"], + ) def test_analysis_common_terms_in_between(self): scores = { - b"simple_sentence": 2, b"sentence_with_many": 2, - b"many_possible": 2, b"many_of_the_possible": 2, b"possible_bigrams": 2} - s = ["sentence", "with", "many", "possible", "bigrams"] - result = self.analyze_words(scores, s) + "simple_sentence": 2, "sentence_with_many": 2, + "many_possible": 2, "many_of_the_possible": 2, "possible_bigrams": 2, + } + sentence = ["sentence", "with", "many", "possible", "bigrams"] + result = self.AnalysisTester(scores, threshold=1)[sentence] self.assertEqual(result, ["sentence_with_many", "possible_bigrams"]) - s = ["a", "simple", "sentence", "with", "many", "of", "the", "possible", "bigrams", "with"] - result = self.analyze_words(scores, s) + sentence = ["a", "simple", "sentence", "with", "many", "of", "the", "possible", "bigrams", "with"] + result = self.AnalysisTester(scores, threshold=1)[sentence] self.assertEqual( result, ["a", "simple_sentence", "with", "many_of_the_possible", "bigrams", "with"]) class PhrasesData: + sentences = common_texts + [ - ['graph', 'minors', 'survey', 'human', 'interface'] + ['graph', 'minors', 'survey', 'human', 'interface'], ] - unicode_sentences = [[to_unicode(w) for w in sentence] for sentence in sentences] common_terms = frozenset() bigram1 = u'response_time' @@ -148,24 +105,18 @@ def gen_sentences(self): return ((w for w in sentence) for sentence in self.sentences) -class PhrasesCommon: - """ Tests that need to be run for both Phrases and Phraser classes.""" +class PhrasesCommon(PhrasesData): + """Tests for both Phrases and FrozenPhrases classes.""" def setUp(self): - self.bigram = Phrases( - self.sentences, min_count=1, threshold=1, common_terms=self.common_terms) - self.bigram_default = Phrases( - self.sentences, common_terms=self.common_terms) - self.bigram_utf8 = Phrases( - self.sentences, min_count=1, threshold=1, common_terms=self.common_terms) - self.bigram_unicode = Phrases( - self.unicode_sentences, min_count=1, threshold=1, common_terms=self.common_terms) + self.bigram = Phrases(self.sentences, min_count=1, threshold=1, common_terms=self.common_terms) + self.bigram_default = Phrases(self.sentences, common_terms=self.common_terms) def testEmptyPhrasifiedSentencesIterator(self): bigram_phrases = Phrases(self.sentences) - bigram_phraser = Phraser(bigram_phrases) + bigram_phraser = FrozenPhrases(bigram_phrases) trigram_phrases = Phrases(bigram_phraser[self.sentences]) - trigram_phraser = Phraser(trigram_phrases) + trigram_phraser = FrozenPhrases(trigram_phrases) trigrams = trigram_phraser[bigram_phraser[self.sentences]] fst, snd = list(trigrams), list(trigrams) self.assertEqual(fst, snd) @@ -187,22 +138,27 @@ def testEmptyInputsOnBigramConstruction(self): def testSentenceGeneration(self): """Test basic bigram using a dummy corpus.""" # test that we generate the same amount of sentences as the input - self.assertEqual(len(self.sentences), len(list(self.bigram_default[self.sentences]))) + self.assertEqual( + len(self.sentences), + len(list(self.bigram_default[self.sentences])), + ) def testSentenceGenerationWithGenerator(self): """Test basic bigram production when corpus is a generator.""" - self.assertEqual(len(list(self.gen_sentences())), - len(list(self.bigram_default[self.gen_sentences()]))) + self.assertEqual( + len(list(self.gen_sentences())), + len(list(self.bigram_default[self.gen_sentences()])), + ) def testBigramConstruction(self): - """Test Phrases bigram construction building.""" + """Test Phrases bigram construction.""" # with this setting we should get response_time and graph_minors bigram1_seen = False bigram2_seen = False - for s in self.bigram[self.sentences]: - if not bigram1_seen and self.bigram1 in s: + for sentence in self.bigram[self.sentences]: + if not bigram1_seen and self.bigram1 in sentence: bigram1_seen = True - if not bigram2_seen and self.bigram2 in s: + if not bigram2_seen and self.bigram2 in sentence: bigram2_seen = True if bigram1_seen and bigram2_seen: break @@ -218,7 +174,7 @@ def testBigramConstruction(self): self.assertTrue(self.bigram3 in self.bigram[self.sentences[-1]]) def testBigramConstructionFromGenerator(self): - """Test Phrases bigram construction building when corpus is a generator""" + """Test Phrases bigram construction building when corpus is a generator.""" bigram1_seen = False bigram2_seen = False @@ -232,7 +188,7 @@ def testBigramConstructionFromGenerator(self): self.assertTrue(bigram1_seen and bigram2_seen) def testBigramConstructionFromArray(self): - """Test Phrases bigram construction building when corpus is a numpy array""" + """Test Phrases bigram construction building when corpus is a numpy array.""" bigram1_seen = False bigram2_seen = False @@ -245,16 +201,6 @@ def testBigramConstructionFromArray(self): break self.assertTrue(bigram1_seen and bigram2_seen) - def testEncoding(self): - """Test that both utf8 and unicode input work; output must be unicode.""" - expected = [u'survey', u'user', u'computer', u'system', u'response_time'] - - self.assertEqual(self.bigram_utf8[self.sentences[1]], expected) - self.assertEqual(self.bigram_unicode[self.sentences[1]], expected) - - transformed = ' '.join(self.bigram_utf8[self.sentences[1]]) - self.assertTrue(isinstance(transformed, six.text_type)) - # scorer for testCustomScorer # function is outside of the scope of the test because for picklability of custom scorer @@ -264,43 +210,32 @@ def dumb_scorer(worda_count, wordb_count, bigram_count, len_vocab, min_count, co return 1 -class TestPhrasesModel(PhrasesData, PhrasesCommon, unittest.TestCase): +class TestPhrasesModel(PhrasesCommon, unittest.TestCase): def testExportPhrases(self): - """Test Phrases bigram export_phrases functionality.""" - bigram = Phrases(self.sentences, min_count=1, threshold=1) - - seen_bigrams = set() - - for phrase, score in bigram.export_phrases(self.sentences): - seen_bigrams.add(phrase) + """Test Phrases bigram export phrases functionality.""" + bigram = Phrases(self.sentences, min_count=1, threshold=1, delimiter=' ') + seen_bigrams = set(phrase for phrase, score in bigram.export_phrases(self.sentences)) assert seen_bigrams == { - b'response time', - b'graph minors', - b'human interface', + 'response time', + 'graph minors', + 'human interface', } def testMultipleBigramsSingleEntry(self): - """ a single entry should produce multiple bigrams. """ - bigram = Phrases(self.sentences, min_count=1, threshold=1) - seen_bigrams = set() - + """Test a single entry produces multiple bigrams.""" + bigram = Phrases(self.sentences, min_count=1, threshold=1, delimiter=' ') test_sentences = [['graph', 'minors', 'survey', 'human', 'interface']] - for phrase, score in bigram.export_phrases(test_sentences): - seen_bigrams.add(phrase) + seen_bigrams = set(phrase for phrase, score in bigram.export_phrases(test_sentences)) - assert seen_bigrams == {b'graph minors', b'human interface'} + assert seen_bigrams == {'graph minors', 'human interface'} def testScoringDefault(self): - """ test the default scoring, from the mikolov word2vec paper """ - bigram = Phrases(self.sentences, min_count=1, threshold=1) - - seen_scores = set() - + """Test the default scoring, from the mikolov word2vec paper.""" + bigram = Phrases(self.sentences, min_count=1, threshold=1, delimiter=' ') test_sentences = [['graph', 'minors', 'survey', 'human', 'interface']] - for phrase, score in bigram.export_phrases(test_sentences): - seen_scores.add(round(score, 3)) + seen_scores = set(round(score, 3) for phrase, score in bigram.export_phrases(test_sentences)) assert seen_scores == { 5.167, # score for graph minors @@ -308,22 +243,18 @@ def testScoringDefault(self): } def test__getitem__(self): - """ test Phrases[sentences] with a single sentence""" + """Test Phrases[sentences] with a single sentence.""" bigram = Phrases(self.sentences, min_count=1, threshold=1) - # pdb.set_trace() test_sentences = [['graph', 'minors', 'survey', 'human', 'interface']] phrased_sentence = next(bigram[test_sentences].__iter__()) assert phrased_sentence == ['graph_minors', 'survey', 'human_interface'] def testScoringNpmi(self): - """ test normalized pointwise mutual information scoring """ + """Test normalized pointwise mutual information scoring.""" bigram = Phrases(self.sentences, min_count=1, threshold=.5, scoring='npmi') - - seen_scores = set() test_sentences = [['graph', 'minors', 'survey', 'human', 'interface']] - for phrase, score in bigram.export_phrases(test_sentences): - seen_scores.add(round(score, 3)) + seen_scores = set(round(score, 3) for phrase, score in bigram.export_phrases(test_sentences)) assert seen_scores == { .882, # score for graph minors @@ -331,16 +262,12 @@ def testScoringNpmi(self): } def testCustomScorer(self): - """ test using a custom scoring function """ - + """Test using a custom scoring function.""" bigram = Phrases(self.sentences, min_count=1, threshold=.001, scoring=dumb_scorer) - - seen_scores = [] test_sentences = [['graph', 'minors', 'survey', 'human', 'interface', 'system']] - for phrase, score in bigram.export_phrases(test_sentences): - seen_scores.append(score) + seen_scores = [score for phrase, score in bigram.export_phrases(test_sentences)] - assert all(seen_scores) # all scores 1 + assert all(score == 1 for score in seen_scores) assert len(seen_scores) == 3 # 'graph minors' and 'survey human' and 'interface system' def testBadParameters(self): @@ -361,31 +288,25 @@ def testPruning(self): class TestPhrasesPersistence(PhrasesData, unittest.TestCase): def testSaveLoadCustomScorer(self): - """ saving and loading a Phrases object with a custom scorer """ - + """Test saving and loading a Phrases object with a custom scorer.""" with temporary_file("test.pkl") as fpath: bigram = Phrases(self.sentences, min_count=1, threshold=.001, scoring=dumb_scorer) bigram.save(fpath) bigram_loaded = Phrases.load(fpath) - seen_scores = [] test_sentences = [['graph', 'minors', 'survey', 'human', 'interface', 'system']] - for phrase, score in bigram_loaded.export_phrases(test_sentences): - seen_scores.append(score) + seen_scores = [score for phrase, score in bigram_loaded.export_phrases(test_sentences)] - assert all(seen_scores) # all scores 1 + assert all(score == 1 for score in seen_scores) assert len(seen_scores) == 3 # 'graph minors' and 'survey human' and 'interface system' def testSaveLoad(self): - """ Saving and loading a Phrases object.""" - + """Test saving and loading a Phrases object.""" with temporary_file("test.pkl") as fpath: bigram = Phrases(self.sentences, min_count=1, threshold=1) bigram.save(fpath) bigram_loaded = Phrases.load(fpath) - seen_scores = set() test_sentences = [['graph', 'minors', 'survey', 'human', 'interface', 'system']] - for phrase, score in bigram_loaded.export_phrases(test_sentences): - seen_scores.add(round(score, 3)) + seen_scores = set(round(score, 3) for phrase, score in bigram_loaded.export_phrases(test_sentences)) assert seen_scores == set([ 5.167, # score for graph minors @@ -393,13 +314,10 @@ def testSaveLoad(self): ]) def testSaveLoadStringScoring(self): - """ Saving and loading a Phrases object with a string scoring parameter. - This should ensure backwards compatibility with the previous version of Phrases""" + """Test backwards compatibility with a previous version of Phrases with custom scoring.""" bigram_loaded = Phrases.load(datapath("phrases-scoring-str.pkl")) - seen_scores = set() test_sentences = [['graph', 'minors', 'survey', 'human', 'interface', 'system']] - for phrase, score in bigram_loaded.export_phrases(test_sentences): - seen_scores.add(round(score, 3)) + seen_scores = set(round(score, 3) for phrase, score in bigram_loaded.export_phrases(test_sentences)) assert seen_scores == set([ 5.167, # score for graph minors @@ -407,14 +325,10 @@ def testSaveLoadStringScoring(self): ]) def testSaveLoadNoScoring(self): - """ Saving and loading a Phrases object with no scoring parameter. - This should ensure backwards compatibility with old versions of Phrases""" - + """Test backwards compatibility with old versions of Phrases with no scoring parameter.""" bigram_loaded = Phrases.load(datapath("phrases-no-scoring.pkl")) - seen_scores = set() test_sentences = [['graph', 'minors', 'survey', 'human', 'interface', 'system']] - for phrase, score in bigram_loaded.export_phrases(test_sentences): - seen_scores.add(round(score, 3)) + seen_scores = set(round(score, 3) for phrase, score in bigram_loaded.export_phrases(test_sentences)) assert seen_scores == set([ 5.167, # score for graph minors @@ -426,77 +340,67 @@ def testSaveLoadNoCommonTerms(self): bigram_loaded = Phrases.load(datapath("phrases-no-common-terms.pkl")) self.assertEqual(bigram_loaded.common_terms, frozenset()) # can make a phraser, cf #1751 - phraser = Phraser(bigram_loaded) # does not raise + phraser = FrozenPhrases(bigram_loaded) # does not raise phraser[["human", "interface", "survey"]] # does not raise -class TestPhraserPersistence(PhrasesData, unittest.TestCase): +class TestFrozenPhrasesPersistence(PhrasesData, unittest.TestCase): def testSaveLoadCustomScorer(self): - """Saving and loading a Phraser object with a custom scorer """ + """Test saving and loading a FrozenPhrases object with a custom scorer.""" with temporary_file("test.pkl") as fpath: - bigram = Phraser( + bigram = FrozenPhrases( Phrases(self.sentences, min_count=1, threshold=.001, scoring=dumb_scorer)) bigram.save(fpath) - bigram_loaded = Phraser.load(fpath) - # we do not much with scoring, just verify its the one expected + bigram_loaded = FrozenPhrases.load(fpath) self.assertEqual(bigram_loaded.scoring, dumb_scorer) def testSaveLoad(self): - """ Saving and loading a Phraser object.""" + """Test saving and loading a FrozenPhrases object.""" with temporary_file("test.pkl") as fpath: - bigram = Phraser(Phrases(self.sentences, min_count=1, threshold=1)) + bigram = FrozenPhrases(Phrases(self.sentences, min_count=1, threshold=1)) bigram.save(fpath) - bigram_loaded = Phraser.load(fpath) + bigram_loaded = FrozenPhrases.load(fpath) self.assertEqual( bigram_loaded[['graph', 'minors', 'survey', 'human', 'interface', 'system']], ['graph_minors', 'survey', 'human_interface', 'system']) def testSaveLoadStringScoring(self): - """ Saving and loading a Phraser object with a string scoring parameter. - This should ensure backwards compatibility with the previous version of Phraser""" - bigram_loaded = Phraser.load(datapath("phraser-scoring-str.pkl")) + """Test saving and loading a FrozenPhrases object with a string scoring parameter. + This should ensure backwards compatibility with the previous version of FrozenPhrases""" + bigram_loaded = FrozenPhrases.load(datapath("phraser-scoring-str.pkl")) # we do not much with scoring, just verify its the one expected self.assertEqual(bigram_loaded.scoring, original_scorer) def testSaveLoadNoScoring(self): - """ Saving and loading a Phraser object with no scoring parameter. - This should ensure backwards compatibility with old versions of Phraser""" - bigram_loaded = Phraser.load(datapath("phraser-no-scoring.pkl")) + """Test saving and loading a FrozenPhrases object with no scoring parameter. + This should ensure backwards compatibility with old versions of FrozenPhrases""" + bigram_loaded = FrozenPhrases.load(datapath("phraser-no-scoring.pkl")) # we do not much with scoring, just verify its the one expected self.assertEqual(bigram_loaded.scoring, original_scorer) def testSaveLoadNoCommonTerms(self): - """ Ensure backwards compatibility with old versions of Phraser, before common_terms""" - bigram_loaded = Phraser.load(datapath("phraser-no-common-terms.pkl")) + """Ensure backwards compatibility with old versions of FrozenPhrases, before common_terms.""" + bigram_loaded = FrozenPhrases.load(datapath("phraser-no-common-terms.pkl")) self.assertEqual(bigram_loaded.common_terms, frozenset()) -class TestPhraserModel(PhrasesData, PhrasesCommon, unittest.TestCase): - """ Test Phraser models.""" +class TestFrozenPhrasesModel(PhrasesCommon, unittest.TestCase): + """Test FrozenPhrases models.""" def setUp(self): - """Set up Phraser models for the tests.""" + """Set up FrozenPhrases models for the tests.""" bigram_phrases = Phrases( self.sentences, min_count=1, threshold=1, common_terms=self.common_terms) - self.bigram = Phraser(bigram_phrases) + self.bigram = FrozenPhrases(bigram_phrases) bigram_default_phrases = Phrases(self.sentences, common_terms=self.common_terms) - self.bigram_default = Phraser(bigram_default_phrases) - - bigram_utf8_phrases = Phrases( - self.sentences, min_count=1, threshold=1, common_terms=self.common_terms) - self.bigram_utf8 = Phraser(bigram_utf8_phrases) - - bigram_unicode_phrases = Phrases( - self.unicode_sentences, min_count=1, threshold=1, common_terms=self.common_terms) - self.bigram_unicode = Phraser(bigram_unicode_phrases) + self.bigram_default = FrozenPhrases(bigram_default_phrases) class CommonTermsPhrasesData: - """This mixin permits to reuse the test, using, this time the common_terms option - """ + """This mixin permits to reuse tests with the common_terms option.""" sentences = [ ['human', 'interface', 'with', 'computer'], @@ -510,7 +414,6 @@ class CommonTermsPhrasesData: ['data', 'and', 'graph', 'survey'], ['data', 'and', 'graph', 'survey', 'for', 'human', 'interface'] # test bigrams within same sentence ] - unicode_sentences = [[to_unicode(w) for w in sentence] for sentence in sentences] common_terms = ['of', 'and', 'for'] bigram1 = u'lack_of_interest' @@ -527,63 +430,43 @@ def gen_sentences(self): class TestPhrasesModelCommonTerms(CommonTermsPhrasesData, TestPhrasesModel): """Test Phrases models with common terms""" - def testEncoding(self): - """Test that both utf8 and unicode input work; output must be unicode.""" - expected = [u'survey', u'of', u'user', u'computer', u'system', u'lack_of_interest'] - - self.assertEqual(self.bigram_utf8[self.sentences[1]], expected) - self.assertEqual(self.bigram_unicode[self.sentences[1]], expected) - - transformed = ' '.join(self.bigram_utf8[self.sentences[1]]) - self.assertTrue(isinstance(transformed, six.text_type)) - def testMultipleBigramsSingleEntry(self): - """ a single entry should produce multiple bigrams. """ - bigram = Phrases(self.sentences, min_count=1, threshold=1, common_terms=self.common_terms) - - seen_bigrams = set() + """Test a single entry produces multiple bigrams.""" + bigram = Phrases(self.sentences, min_count=1, threshold=1, common_terms=self.common_terms, delimiter=' ') test_sentences = [['data', 'and', 'graph', 'survey', 'for', 'human', 'interface']] - for phrase, score in bigram.export_phrases(test_sentences): - seen_bigrams.add(phrase) + seen_bigrams = set(phrase for phrase, score in bigram.export_phrases(test_sentences)) + assert seen_bigrams == set([ - b'data and graph', - b'human interface', + 'data and graph', + 'human interface', ]) def testExportPhrases(self): - """Test Phrases bigram export_phrases functionality.""" - bigram = Phrases(self.sentences, min_count=1, threshold=1, common_terms=self.common_terms) - - seen_bigrams = set() - - for phrase, score in bigram.export_phrases(self.sentences): - seen_bigrams.add(phrase) + """Test Phrases bigram export phrases functionality.""" + bigram = Phrases(self.sentences, min_count=1, threshold=1, common_terms=self.common_terms, delimiter=' ') + seen_bigrams = set(phrase for phrase, score in bigram.export_phrases(self.sentences)) assert seen_bigrams == set([ - b'human interface', - b'graph of trees', - b'data and graph', - b'lack of interest', + 'human interface', + 'graph of trees', + 'data and graph', + 'lack of interest', ]) def testScoringDefault(self): """ test the default scoring, from the mikolov word2vec paper """ bigram = Phrases(self.sentences, min_count=1, threshold=1, common_terms=self.common_terms) - - seen_scores = set() - test_sentences = [['data', 'and', 'graph', 'survey', 'for', 'human', 'interface']] - for phrase, score in bigram.export_phrases(test_sentences): - seen_scores.add(round(score, 3)) + seen_scores = set(round(score, 3) for phrase, score in bigram.export_phrases(test_sentences)) min_count = float(bigram.min_count) len_vocab = float(len(bigram.vocab)) - graph = float(bigram.vocab[b"graph"]) - data = float(bigram.vocab[b"data"]) - data_and_graph = float(bigram.vocab[b"data_and_graph"]) - human = float(bigram.vocab[b"human"]) - interface = float(bigram.vocab[b"interface"]) - human_interface = float(bigram.vocab[b"human_interface"]) + graph = float(bigram.vocab["graph"]) + data = float(bigram.vocab["data"]) + data_and_graph = float(bigram.vocab["data_and_graph"]) + human = float(bigram.vocab["human"]) + interface = float(bigram.vocab["interface"]) + human_interface = float(bigram.vocab["human_interface"]) assert seen_scores == set([ # score for data and graph @@ -593,15 +476,13 @@ def testScoringDefault(self): ]) def testScoringNpmi(self): - """ test normalized pointwise mutual information scoring """ - bigram = Phrases(self.sentences, min_count=1, threshold=.5, - scoring='npmi', common_terms=self.common_terms) - - seen_scores = set() - + """Test normalized pointwise mutual information scoring.""" + bigram = Phrases( + self.sentences, min_count=1, threshold=.5, + scoring='npmi', common_terms=self.common_terms, + ) test_sentences = [['data', 'and', 'graph', 'survey', 'for', 'human', 'interface']] - for phrase, score in bigram.export_phrases(test_sentences): - seen_scores.add(round(score, 3)) + seen_scores = set(round(score, 3) for phrase, score in bigram.export_phrases(test_sentences)) assert seen_scores == set([ .74, # score for data and graph @@ -609,56 +490,35 @@ def testScoringNpmi(self): ]) def testCustomScorer(self): - """ test using a custom scoring function """ - - bigram = Phrases(self.sentences, min_count=1, threshold=.001, - scoring=dumb_scorer, common_terms=self.common_terms) - - seen_scores = [] + """Test using a custom scoring function.""" + bigram = Phrases( + self.sentences, min_count=1, threshold=.001, + scoring=dumb_scorer, common_terms=self.common_terms, + ) test_sentences = [['data', 'and', 'graph', 'survey', 'for', 'human', 'interface']] - for phrase, score in bigram.export_phrases(test_sentences): - seen_scores.append(score) + seen_scores = [score for phrase, score in bigram.export_phrases(test_sentences)] assert all(seen_scores) # all scores 1 assert len(seen_scores) == 2 # 'data and graph' 'survey for human' def test__getitem__(self): - """ test Phrases[sentences] with a single sentence""" + """Test Phrases[sentences] with a single sentence.""" bigram = Phrases(self.sentences, min_count=1, threshold=1, common_terms=self.common_terms) - # pdb.set_trace() test_sentences = [['data', 'and', 'graph', 'survey', 'for', 'human', 'interface']] phrased_sentence = next(bigram[test_sentences].__iter__()) assert phrased_sentence == ['data_and_graph', 'survey', 'for', 'human_interface'] -class TestPhraserModelCommonTerms(CommonTermsPhrasesData, TestPhraserModel): - - def testEncoding(self): - """Test that both utf8 and unicode input work; output must be unicode.""" - expected = [u'survey', u'of', u'user', u'computer', u'system', u'lack_of_interest'] - - self.assertEqual(self.bigram_utf8[self.sentences[1]], expected) - self.assertEqual(self.bigram_unicode[self.sentences[1]], expected) - - transformed = ' '.join(self.bigram_utf8[self.sentences[1]]) - self.assertTrue(isinstance(transformed, six.text_type)) - - -class TestPhraserModelCompatibilty(unittest.TestCase): +class TestFrozenPhrasesModelCompatibilty(unittest.TestCase): def testCompatibilty(self): - phr = Phraser.load(datapath("phraser-3.6.0.model")) - model = Phrases.load(datapath("phrases-3.6.0.model")) - + phrases = Phrases.load(datapath("phrases-3.6.0.model")) + phraser = FrozenPhrases.load(datapath("phraser-3.6.0.model")) test_sentences = ['trees', 'graph', 'minors'] - expected_res = ['trees', 'graph_minors'] - - phr_out = phr[test_sentences] - model_out = model[test_sentences] - self.assertEqual(phr_out, expected_res) - self.assertEqual(model_out, expected_res) + self.assertEqual(phrases[test_sentences], ['trees', 'graph_minors']) + self.assertEqual(phraser[test_sentences], ['trees', 'graph_minors']) if __name__ == '__main__': From 092b5129c826a7fb88f1afbc43587bcd087d9e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Radim=20=C5=98eh=C5=AF=C5=99ek?= Date: Sat, 10 Oct 2020 11:23:49 +0200 Subject: [PATCH 3/5] clean up rendered docs for phrases --- docs/src/auto_examples/index.rst | 88 ++++++++++++++++++-------------- gensim/models/phrases.py | 31 ++++------- gensim/test/test_phrases.py | 32 ++++++------ 3 files changed, 77 insertions(+), 74 deletions(-) diff --git a/docs/src/auto_examples/index.rst b/docs/src/auto_examples/index.rst index cbe06b6fb5..ca3c1ec019 100644 --- a/docs/src/auto_examples/index.rst +++ b/docs/src/auto_examples/index.rst @@ -13,7 +13,7 @@ If you're thinking about contributing documentation, please see :ref:`sphx_glr_a .. raw:: html -
+
@@ -33,9 +33,10 @@ Understanding this functionality is vital for using gensim effectively. .. only:: html - .. figure:: /auto_examples/core/images/thumb/sphx_glr_run_core_concepts_thumb.png + .. figure:: /auto_examples/core/images/thumb/sphx_glr_run_core_concepts_thumb.png + :alt: Core Concepts - :ref:`sphx_glr_auto_examples_core_run_core_concepts.py` + :ref:`sphx_glr_auto_examples_core_run_core_concepts.py` .. raw:: html @@ -53,9 +54,10 @@ Understanding this functionality is vital for using gensim effectively. .. only:: html - .. figure:: /auto_examples/core/images/thumb/sphx_glr_run_corpora_and_vector_spaces_thumb.png + .. figure:: /auto_examples/core/images/thumb/sphx_glr_run_corpora_and_vector_spaces_thumb.png + :alt: Corpora and Vector Spaces - :ref:`sphx_glr_auto_examples_core_run_corpora_and_vector_spaces.py` + :ref:`sphx_glr_auto_examples_core_run_corpora_and_vector_spaces.py` .. raw:: html @@ -73,9 +75,10 @@ Understanding this functionality is vital for using gensim effectively. .. only:: html - .. figure:: /auto_examples/core/images/thumb/sphx_glr_run_topics_and_transformations_thumb.png + .. figure:: /auto_examples/core/images/thumb/sphx_glr_run_topics_and_transformations_thumb.png + :alt: Topics and Transformations - :ref:`sphx_glr_auto_examples_core_run_topics_and_transformations.py` + :ref:`sphx_glr_auto_examples_core_run_topics_and_transformations.py` .. raw:: html @@ -93,9 +96,10 @@ Understanding this functionality is vital for using gensim effectively. .. only:: html - .. figure:: /auto_examples/core/images/thumb/sphx_glr_run_similarity_queries_thumb.png + .. figure:: /auto_examples/core/images/thumb/sphx_glr_run_similarity_queries_thumb.png + :alt: Similarity Queries - :ref:`sphx_glr_auto_examples_core_run_similarity_queries.py` + :ref:`sphx_glr_auto_examples_core_run_similarity_queries.py` .. raw:: html @@ -108,7 +112,7 @@ Understanding this functionality is vital for using gensim effectively. /auto_examples/core/run_similarity_queries .. raw:: html -
+
@@ -127,9 +131,10 @@ Learning-oriented lessons that introduce a particular gensim feature, e.g. a mod .. only:: html - .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_word2vec_thumb.png + .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_word2vec_thumb.png + :alt: Word2Vec Model - :ref:`sphx_glr_auto_examples_tutorials_run_word2vec.py` + :ref:`sphx_glr_auto_examples_tutorials_run_word2vec.py` .. raw:: html @@ -147,9 +152,10 @@ Learning-oriented lessons that introduce a particular gensim feature, e.g. a mod .. only:: html - .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_doc2vec_lee_thumb.png + .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_doc2vec_lee_thumb.png + :alt: Doc2Vec Model - :ref:`sphx_glr_auto_examples_tutorials_run_doc2vec_lee.py` + :ref:`sphx_glr_auto_examples_tutorials_run_doc2vec_lee.py` .. raw:: html @@ -167,9 +173,10 @@ Learning-oriented lessons that introduce a particular gensim feature, e.g. a mod .. only:: html - .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_fasttext_thumb.png + .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_fasttext_thumb.png + :alt: FastText Model - :ref:`sphx_glr_auto_examples_tutorials_run_fasttext.py` + :ref:`sphx_glr_auto_examples_tutorials_run_fasttext.py` .. raw:: html @@ -187,9 +194,10 @@ Learning-oriented lessons that introduce a particular gensim feature, e.g. a mod .. only:: html - .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_annoy_thumb.png + .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_annoy_thumb.png + :alt: Fast Similarity Queries with Annoy and Word2Vec - :ref:`sphx_glr_auto_examples_tutorials_run_annoy.py` + :ref:`sphx_glr_auto_examples_tutorials_run_annoy.py` .. raw:: html @@ -207,9 +215,10 @@ Learning-oriented lessons that introduce a particular gensim feature, e.g. a mod .. only:: html - .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_lda_thumb.png + .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_lda_thumb.png + :alt: LDA Model - :ref:`sphx_glr_auto_examples_tutorials_run_lda.py` + :ref:`sphx_glr_auto_examples_tutorials_run_lda.py` .. raw:: html @@ -227,9 +236,10 @@ Learning-oriented lessons that introduce a particular gensim feature, e.g. a mod .. only:: html - .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_wmd_thumb.png + .. figure:: /auto_examples/tutorials/images/thumb/sphx_glr_run_wmd_thumb.png + :alt: Word Mover's Distance - :ref:`sphx_glr_auto_examples_tutorials_run_wmd.py` + :ref:`sphx_glr_auto_examples_tutorials_run_wmd.py` .. raw:: html @@ -242,7 +252,7 @@ Learning-oriented lessons that introduce a particular gensim feature, e.g. a mod /auto_examples/tutorials/run_wmd .. raw:: html -
+
@@ -261,9 +271,10 @@ These **goal-oriented guides** demonstrate how to **solve a specific problem** u .. only:: html - .. figure:: /auto_examples/howtos/images/thumb/sphx_glr_run_downloader_api_thumb.png + .. figure:: /auto_examples/howtos/images/thumb/sphx_glr_run_downloader_api_thumb.png + :alt: How to download pre-trained models and corpora - :ref:`sphx_glr_auto_examples_howtos_run_downloader_api.py` + :ref:`sphx_glr_auto_examples_howtos_run_downloader_api.py` .. raw:: html @@ -281,9 +292,10 @@ These **goal-oriented guides** demonstrate how to **solve a specific problem** u .. only:: html - .. figure:: /auto_examples/howtos/images/thumb/sphx_glr_run_doc_thumb.png + .. figure:: /auto_examples/howtos/images/thumb/sphx_glr_run_doc_thumb.png + :alt: How to Author Gensim Documentation - :ref:`sphx_glr_auto_examples_howtos_run_doc.py` + :ref:`sphx_glr_auto_examples_howtos_run_doc.py` .. raw:: html @@ -301,9 +313,10 @@ These **goal-oriented guides** demonstrate how to **solve a specific problem** u .. only:: html - .. figure:: /auto_examples/howtos/images/thumb/sphx_glr_run_doc2vec_imdb_thumb.png + .. figure:: /auto_examples/howtos/images/thumb/sphx_glr_run_doc2vec_imdb_thumb.png + :alt: How to reproduce the doc2vec 'Paragraph Vector' paper - :ref:`sphx_glr_auto_examples_howtos_run_doc2vec_imdb.py` + :ref:`sphx_glr_auto_examples_howtos_run_doc2vec_imdb.py` .. raw:: html @@ -321,9 +334,10 @@ These **goal-oriented guides** demonstrate how to **solve a specific problem** u .. only:: html - .. figure:: /auto_examples/howtos/images/thumb/sphx_glr_run_compare_lda_thumb.png + .. figure:: /auto_examples/howtos/images/thumb/sphx_glr_run_compare_lda_thumb.png + :alt: How to Compare LDA Models - :ref:`sphx_glr_auto_examples_howtos_run_compare_lda.py` + :ref:`sphx_glr_auto_examples_howtos_run_compare_lda.py` .. raw:: html @@ -336,7 +350,7 @@ These **goal-oriented guides** demonstrate how to **solve a specific problem** u /auto_examples/howtos/run_compare_lda .. raw:: html -
+
@@ -379,7 +393,7 @@ Blog posts, tutorial videos, hackathons and other useful Gensim resources, from .. raw:: html -
+
@@ -389,15 +403,15 @@ Blog posts, tutorial videos, hackathons and other useful Gensim resources, from :class: sphx-glr-footer-gallery - .. container:: sphx-glr-download + .. container:: sphx-glr-download sphx-glr-download-python - :download:`Download all examples in Python source code: auto_examples_python.zip ` + :download:`Download all examples in Python source code: auto_examples_python.zip ` - .. container:: sphx-glr-download + .. container:: sphx-glr-download sphx-glr-download-jupyter - :download:`Download all examples in Jupyter notebooks: auto_examples_jupyter.zip ` + :download:`Download all examples in Jupyter notebooks: auto_examples_jupyter.zip ` .. only:: html diff --git a/gensim/models/phrases.py b/gensim/models/phrases.py index 43806d36a9..8bf4b57ea4 100644 --- a/gensim/models/phrases.py +++ b/gensim/models/phrases.py @@ -49,7 +49,7 @@ >>> >>> # Export the trained model = use less RAM, faster processing. Model updates no longer possible. >>> frozen_model = phrase_model.freeze() - >>> # Apply the exported model to a sentence; same results as before: + >>> # Apply the frozen model; same results as before: >>> frozen_model[new_sentence] ['trees_graph', 'minors'] >>> @@ -294,7 +294,7 @@ def __getitem__(self, sentence): return [token for token, _ in self.analyze_sentence(sentence)] def export_phrases(self, sentences): - """Get all unique phrases (multi-word expressions) that appear in 'sentences'. + """Get all unique phrases (multi-word expressions) that appear in ``sentences``, and their scores. Parameters ---------- @@ -303,12 +303,13 @@ def export_phrases(self, sentences): Returns ------- - set((str, float)) - Set of unique phrases and their scores. + dict(str, float) + Unique phrases mapped to their scores. Example ------- .. sourcecode:: pycon + >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus >>> from gensim.models.phrases import Phrases @@ -316,14 +317,14 @@ def export_phrases(self, sentences): >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> phrases = Phrases(sentences, min_count=1, threshold=0.1) >>> - >>> for phrase, score in phrases.export_phrases(sentences): + >>> for phrase, score in phrases.export_phrases(sentences).items(): ... print(phrase, score) """ - result = set() + result = {} for sentence in sentences: for phrase, score in self.analyze_sentence(sentence): if score is not None: - result.add((phrase, score)) + result[phrase] = score return result @classmethod @@ -731,7 +732,7 @@ def __init__(self, phrases_model): Notes ----- - After the one-time initialization, a :class:`~gensim.models.phrases.FrozenPhrases` will be much smaller and somewhat + After the one-time initialization, a :class:`~gensim.models.phrases.FrozenPhrases` will be much smaller and faster than using the full :class:`~gensim.models.phrases.Phrases` model. Examples @@ -746,19 +747,7 @@ def __init__(self, phrases_model): >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> phrases = Phrases(sentences, min_count=1, threshold=1) >>> - >>> # Use the model to detect phrases in a new sentence. - >>> sent = [u'trees', u'graph', u'minors'] - >>> print(phrases[sent]) - [u'trees_graph', u'minors'] - >>> - >>> # Or transform multiple sentences at once. - >>> sents = [[u'trees', u'graph', u'minors'], [u'graph', u'minors']] - >>> for phrase in frozen_phrases[sents]: - ... print(phrase) - [u'trees_graph', u'minors'] - [u'graph_minors'] - >>> - >>> # Export a FrozenPhrases object that is more efficient but doesn't allow any more training. + >>> # Export FrozenPhrases that is more efficient but doesn't allow any more training. >>> frozen_phrases = phrases.freeze() >>> print(frozen_phrases[sent]) [u'trees_graph', u'minors'] diff --git a/gensim/test/test_phrases.py b/gensim/test/test_phrases.py index cec2747073..9c7a73cae4 100644 --- a/gensim/test/test_phrases.py +++ b/gensim/test/test_phrases.py @@ -213,9 +213,9 @@ def dumb_scorer(worda_count, wordb_count, bigram_count, len_vocab, min_count, co class TestPhrasesModel(PhrasesCommon, unittest.TestCase): def testExportPhrases(self): - """Test Phrases bigram export phrases functionality.""" + """Test Phrases bigram export phrases.""" bigram = Phrases(self.sentences, min_count=1, threshold=1, delimiter=' ') - seen_bigrams = set(phrase for phrase, score in bigram.export_phrases(self.sentences)) + seen_bigrams = set(bigram.export_phrases(self.sentences).keys()) assert seen_bigrams == { 'response time', @@ -227,7 +227,7 @@ def testMultipleBigramsSingleEntry(self): """Test a single entry produces multiple bigrams.""" bigram = Phrases(self.sentences, min_count=1, threshold=1, delimiter=' ') test_sentences = [['graph', 'minors', 'survey', 'human', 'interface']] - seen_bigrams = set(phrase for phrase, score in bigram.export_phrases(test_sentences)) + seen_bigrams = set(bigram.export_phrases(test_sentences).keys()) assert seen_bigrams == {'graph minors', 'human interface'} @@ -235,7 +235,7 @@ def testScoringDefault(self): """Test the default scoring, from the mikolov word2vec paper.""" bigram = Phrases(self.sentences, min_count=1, threshold=1, delimiter=' ') test_sentences = [['graph', 'minors', 'survey', 'human', 'interface']] - seen_scores = set(round(score, 3) for phrase, score in bigram.export_phrases(test_sentences)) + seen_scores = set(round(score, 3) for score in bigram.export_phrases(test_sentences).values()) assert seen_scores == { 5.167, # score for graph minors @@ -254,7 +254,7 @@ def testScoringNpmi(self): """Test normalized pointwise mutual information scoring.""" bigram = Phrases(self.sentences, min_count=1, threshold=.5, scoring='npmi') test_sentences = [['graph', 'minors', 'survey', 'human', 'interface']] - seen_scores = set(round(score, 3) for phrase, score in bigram.export_phrases(test_sentences)) + seen_scores = set(round(score, 3) for score in bigram.export_phrases(test_sentences).values()) assert seen_scores == { .882, # score for graph minors @@ -265,7 +265,7 @@ def testCustomScorer(self): """Test using a custom scoring function.""" bigram = Phrases(self.sentences, min_count=1, threshold=.001, scoring=dumb_scorer) test_sentences = [['graph', 'minors', 'survey', 'human', 'interface', 'system']] - seen_scores = [score for phrase, score in bigram.export_phrases(test_sentences)] + seen_scores = list(bigram.export_phrases(test_sentences).values()) assert all(score == 1 for score in seen_scores) assert len(seen_scores) == 3 # 'graph minors' and 'survey human' and 'interface system' @@ -294,7 +294,7 @@ def testSaveLoadCustomScorer(self): bigram.save(fpath) bigram_loaded = Phrases.load(fpath) test_sentences = [['graph', 'minors', 'survey', 'human', 'interface', 'system']] - seen_scores = [score for phrase, score in bigram_loaded.export_phrases(test_sentences)] + seen_scores = list(bigram_loaded.export_phrases(test_sentences).values()) assert all(score == 1 for score in seen_scores) assert len(seen_scores) == 3 # 'graph minors' and 'survey human' and 'interface system' @@ -306,7 +306,7 @@ def testSaveLoad(self): bigram.save(fpath) bigram_loaded = Phrases.load(fpath) test_sentences = [['graph', 'minors', 'survey', 'human', 'interface', 'system']] - seen_scores = set(round(score, 3) for phrase, score in bigram_loaded.export_phrases(test_sentences)) + seen_scores = set(round(score, 3) for score in bigram_loaded.export_phrases(test_sentences).values()) assert seen_scores == set([ 5.167, # score for graph minors @@ -317,7 +317,7 @@ def testSaveLoadStringScoring(self): """Test backwards compatibility with a previous version of Phrases with custom scoring.""" bigram_loaded = Phrases.load(datapath("phrases-scoring-str.pkl")) test_sentences = [['graph', 'minors', 'survey', 'human', 'interface', 'system']] - seen_scores = set(round(score, 3) for phrase, score in bigram_loaded.export_phrases(test_sentences)) + seen_scores = set(round(score, 3) for score in bigram_loaded.export_phrases(test_sentences).values()) assert seen_scores == set([ 5.167, # score for graph minors @@ -328,7 +328,7 @@ def testSaveLoadNoScoring(self): """Test backwards compatibility with old versions of Phrases with no scoring parameter.""" bigram_loaded = Phrases.load(datapath("phrases-no-scoring.pkl")) test_sentences = [['graph', 'minors', 'survey', 'human', 'interface', 'system']] - seen_scores = set(round(score, 3) for phrase, score in bigram_loaded.export_phrases(test_sentences)) + seen_scores = set(round(score, 3) for score in bigram_loaded.export_phrases(test_sentences).values()) assert seen_scores == set([ 5.167, # score for graph minors @@ -434,7 +434,7 @@ def testMultipleBigramsSingleEntry(self): """Test a single entry produces multiple bigrams.""" bigram = Phrases(self.sentences, min_count=1, threshold=1, common_terms=self.common_terms, delimiter=' ') test_sentences = [['data', 'and', 'graph', 'survey', 'for', 'human', 'interface']] - seen_bigrams = set(phrase for phrase, score in bigram.export_phrases(test_sentences)) + seen_bigrams = set(bigram.export_phrases(test_sentences).keys()) assert seen_bigrams == set([ 'data and graph', @@ -442,9 +442,9 @@ def testMultipleBigramsSingleEntry(self): ]) def testExportPhrases(self): - """Test Phrases bigram export phrases functionality.""" + """Test Phrases bigram export phrases.""" bigram = Phrases(self.sentences, min_count=1, threshold=1, common_terms=self.common_terms, delimiter=' ') - seen_bigrams = set(phrase for phrase, score in bigram.export_phrases(self.sentences)) + seen_bigrams = set(bigram.export_phrases(self.sentences).keys()) assert seen_bigrams == set([ 'human interface', @@ -457,7 +457,7 @@ def testScoringDefault(self): """ test the default scoring, from the mikolov word2vec paper """ bigram = Phrases(self.sentences, min_count=1, threshold=1, common_terms=self.common_terms) test_sentences = [['data', 'and', 'graph', 'survey', 'for', 'human', 'interface']] - seen_scores = set(round(score, 3) for phrase, score in bigram.export_phrases(test_sentences)) + seen_scores = set(round(score, 3) for score in bigram.export_phrases(test_sentences).values()) min_count = float(bigram.min_count) len_vocab = float(len(bigram.vocab)) @@ -482,7 +482,7 @@ def testScoringNpmi(self): scoring='npmi', common_terms=self.common_terms, ) test_sentences = [['data', 'and', 'graph', 'survey', 'for', 'human', 'interface']] - seen_scores = set(round(score, 3) for phrase, score in bigram.export_phrases(test_sentences)) + seen_scores = set(round(score, 3) for score in bigram.export_phrases(test_sentences).values()) assert seen_scores == set([ .74, # score for data and graph @@ -496,7 +496,7 @@ def testCustomScorer(self): scoring=dumb_scorer, common_terms=self.common_terms, ) test_sentences = [['data', 'and', 'graph', 'survey', 'for', 'human', 'interface']] - seen_scores = [score for phrase, score in bigram.export_phrases(test_sentences)] + seen_scores = list(bigram.export_phrases(test_sentences).values()) assert all(seen_scores) # all scores 1 assert len(seen_scores) == 2 # 'data and graph' 'survey for human' From 1acb47c6d1aae3f13e080b83d05351ec463237a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Radim=20=C5=98eh=C5=AF=C5=99ek?= Date: Sat, 10 Oct 2020 13:22:33 +0200 Subject: [PATCH 4/5] fix sklearn_api.phrases tests + docs - removed testing of loading of old models for backward compatibility, because the wrappers use plain pickle and so don't support SaveLoad overrides --- gensim/sklearn_api/phrases.py | 20 ++++++----- .../phrases-transformer-new-v3-5-0.pkl | Bin 1896 -> 0 bytes .../test_data/phrases-transformer-v3-5-0.pkl | Bin 1432 -> 0 bytes gensim/test/test_sklearn_api.py | 31 ++---------------- 4 files changed, 13 insertions(+), 38 deletions(-) delete mode 100644 gensim/test/test_data/phrases-transformer-new-v3-5-0.pkl delete mode 100644 gensim/test/test_data/phrases-transformer-v3-5-0.pkl diff --git a/gensim/sklearn_api/phrases.py b/gensim/sklearn_api/phrases.py index 1570acf224..4c04292473 100644 --- a/gensim/sklearn_api/phrases.py +++ b/gensim/sklearn_api/phrases.py @@ -27,12 +27,12 @@ >>> assert ['I', 'love', 'computer_science'] == m.fit_transform(texts)[0] """ -from six import string_types + from sklearn.base import TransformerMixin, BaseEstimator from sklearn.exceptions import NotFittedError from gensim import models -from gensim.models.phrases import Phraser +from gensim.models.phrases import FrozenPhrases class PhrasesTransformer(TransformerMixin, BaseEstimator): @@ -44,8 +44,10 @@ class PhrasesTransformer(TransformerMixin, BaseEstimator): `_. """ - def __init__(self, min_count=5, threshold=10.0, max_vocab_size=40000000, - delimiter=b'_', progress_per=10000, scoring='default', common_terms=frozenset()): + def __init__( + self, min_count=5, threshold=10.0, max_vocab_size=40000000, + delimiter='_', progress_per=10000, scoring='default', common_terms=frozenset(), + ): """ Parameters @@ -58,7 +60,7 @@ def __init__(self, min_count=5, threshold=10.0, max_vocab_size=40000000, Maximum size of the vocabulary. Used to control pruning of less common words, to keep memory under control. The default of 40M needs about 3.6GB of RAM. delimiter : str, optional - Character used to join collocation tokens, should be a byte string (e.g. b'_'). + Character used to join collocation tokens (e.g. '_'). progress_per : int, optional Training will report to the logger every that many phrases are learned. scoring : str or function, optional @@ -127,7 +129,7 @@ def fit(self, X, y=None): max_vocab_size=self.max_vocab_size, delimiter=self.delimiter, progress_per=self.progress_per, scoring=self.scoring, common_terms=self.common_terms ) - self.phraser = Phraser(self.gensim_model) + self.phraser = FrozenPhrases(self.gensim_model) return self def transform(self, docs): @@ -152,10 +154,10 @@ def transform(self, docs): ) if self.phraser is None: - self.phraser = Phraser(self.gensim_model) + self.phraser = FrozenPhrases(self.gensim_model) # input as python lists - if isinstance(docs[0], string_types): + if isinstance(docs[0], str): docs = [docs] return [self.phraser[doc] for doc in docs] @@ -186,5 +188,5 @@ def partial_fit(self, X): ) self.gensim_model.add_vocab(X) - self.phraser = Phraser(self.gensim_model) + self.phraser = FrozenPhrases(self.gensim_model) return self diff --git a/gensim/test/test_data/phrases-transformer-new-v3-5-0.pkl b/gensim/test/test_data/phrases-transformer-new-v3-5-0.pkl deleted file mode 100644 index 77994180586f35f8f185943fc7a1c191dbd29c18..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1896 zcmZ`)hjtrP5S3-8t)0XPp_f1?aVP>O0Zcd55D^GN01*g})#^#RWQ*=TtrHtW62PK| z-g_^<#g8!acC|)6WRK2~G@b)Ih21NPTknF zlQhz)gUX)M80(`t)7Vt0RH7hOUXsN_jKBJJ&h{_*+sZ= zfU8Cbt}X~T*RGFiL~S==rf^?GwMWF)60RHI`uh07{EolvGr$d*^}}TkqJVv&#)f0; zS>Z;pxrs1qdu6;N4dM>=o~2iec7oUql_*N{_cf-N#!I3oi4`+*BlderHM1b(8Y|_r z(`1?FN%tJwd=$5=;8wz1)?Cxj+lpBk`oG;q)6#7>3wyZ3I_aiKhZ|{BmlL@2;Pmds z7Iq!EC)8r%xlLt)WsSQIRE{*OYqGuDpS2A6U@JR+xaTY>79;Lpp2zXtKJJ_Gc>w*r z{yu-dKj+s9u90Z&muRHFHWEDGc}W;*uNUypIYkz&z~dS`I8vN2i8~G+TEPP0VQJ>p zyqMjLOR3C~>FEfMNQVv(>Kw&XV>4@HhS74S9~Jx{;W3L#J90yn58LAcA0j+q;cc)`eL_$l=d4y2?j?~Q zBZO8i3FT(JRBMAs(3p@|r(%`MyCMf6E!pzzGa~O1vibyju?4s5Ule>%;Nyf7maaT4 zOCnw-ylg|CW-n#h!%n{1I0G`=BxYqOS`+vy%w?mL0MC;U*tMIL!4 ze-!v9!p{~K{@l9k0DcjTUkSg}tC=samu;-XH?Er*bv#L1#XIfy`YyXH%+3qg%J^je Z!OH)0i2k&Ph!35_I&vTX^7;PN{slS=U%&tW diff --git a/gensim/test/test_data/phrases-transformer-v3-5-0.pkl b/gensim/test/test_data/phrases-transformer-v3-5-0.pkl deleted file mode 100644 index 8ffef6763b539975f1035f50d458da257b6e183c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1432 zcmZ{kYj+ey5QcY?kidjXz#HBHF9g&m;9U_F5M)KML>y6UXJ>nN2Ils5&jb?X9Q**k zo2r`GiE@sgX1AWI>Z-S@`@upyvxQId4Sy%Ikt9)n*D}$Tk z1x;HgTt-r66YLue)sm+L=WmJ{``IGJww;=s&&qIM-njntPxc&OjT=39ABWbHrT=l| zCTD38Wk&iHZJr8u@iAH;o{r-(%WPbyW#NNPgGn0K0iKDC88>Oh?HLnfWiboz>_a?9 zIJ};$gu6KhH&bz(#q$aG{{rE~(E=k$pEP+?;3cuYOn7Cq$o2zsyGaX{b{fSNuZqVJ z!qGl*=AtUWF|of!czv|Qk*gf3c2hfRJ>C%dCgJ#K$W&Bg9ZTmsye{|z;iO_Irww~c z>~9m^(Q;&ZzHys&3mXF7CA_C#NdnT`;*^+A6V7OHhuGW~&O^(qMi%6Kar%Jpp&ll& zjc4;mV*Z%$iKY~_`iM^jeMb0P(@OPz%hxtHt!QTje?d5>2^SgL%HvD1ent3NgVxx( zQ%f6;^Wt)WaIsfK6PgZxBlr^GvSL|ODfC~n_*NXQ5Ju_{?9?Uae<$!N;d_Pq7pP-rxA))QoZ~`;PIOu-^%{ zG^xaDFTKQn2sea3wOU!cIhv~xiN}}__a5C4=0#v49+ogw551bQnx^Y(CO9Fa{eW2} zqqDy)=nf(4wbJVP%(?J_P%4+@ZbfXPE1@8`{@jkX>){30gl05slI3JI(FaYY#f2GX zCk Date: Sat, 10 Oct 2020 13:57:40 +0200 Subject: [PATCH 5/5] fix flake8 warnings in docstrings --- gensim/models/phrases.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/gensim/models/phrases.py b/gensim/models/phrases.py index 8bf4b57ea4..959604a4fc 100644 --- a/gensim/models/phrases.py +++ b/gensim/models/phrases.py @@ -21,7 +21,7 @@ >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus - >>> from gensim.models.phrases import Phrases, FrozenPhrases + >>> from gensim.models.phrases import Phrases >>> >>> # Create training corpus. Must be a sequence of sentences (e.g. an iterable or a generator). >>> sentences = Text8Corpus(datapath('testcorpus.txt')) @@ -252,7 +252,7 @@ def analyze_sentence(self, sentence): else: # Not inside a potential bigram yet; start a new potential bigram here. start_token, in_between = word, [] - else: # We're a stop word. + else: # We're a stop word. if start_token: # We're inside a potential bigram: add the stopword and keep growing the phrase. in_between.append(word) @@ -475,7 +475,7 @@ def __init__( >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus - >>> from gensim.models.phrases import Phrases, FrozenPhrases + >>> from gensim.models.phrases import Phrases >>> >>> # Load corpus and train a model. >>> sentences = Text8Corpus(datapath('testcorpus.txt')) @@ -732,8 +732,8 @@ def __init__(self, phrases_model): Notes ----- - After the one-time initialization, a :class:`~gensim.models.phrases.FrozenPhrases` will be much smaller and - faster than using the full :class:`~gensim.models.phrases.Phrases` model. + After the one-time initialization, a :class:`~gensim.models.phrases.FrozenPhrases` will be much + smaller and faster than using the full :class:`~gensim.models.phrases.Phrases` model. Examples ---------- @@ -741,13 +741,13 @@ def __init__(self, phrases_model): >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus - >>> from gensim.models.phrases import Phrases, FrozenPhrases + >>> from gensim.models.phrases import Phrases >>> >>> # Load corpus and train a model. >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> phrases = Phrases(sentences, min_count=1, threshold=1) >>> - >>> # Export FrozenPhrases that is more efficient but doesn't allow any more training. + >>> # Export a FrozenPhrases object that is more efficient but doesn't allow further training. >>> frozen_phrases = phrases.freeze() >>> print(frozen_phrases[sent]) [u'trees_graph', u'minors']