#include <dictionary.h>

Public Member Functions
	Dictionary (std::shared_ptr< Args >)

int32_t	nwords () const

int32_t	nlabels () const

int64_t	ntokens () const

int32_t	getId (const std::string &) const

entry_type	getType (int32_t) const

entry_type	getType (const std::string &) const

bool	discard (int32_t, real) const

std::string	getWord (int32_t) const

const std::vector< int32_t > &	getSubwords (int32_t) const

const std::vector< int32_t >	getSubwords (const std::string &) const

void	computeSubwords (const std::string &, std::vector< int32_t > &) const

void	computeSubwords (const std::string &, std::vector< int32_t > &, std::vector< std::string > &) const

void	getSubwords (const std::string &, std::vector< int32_t > &, std::vector< std::string > &) const

uint32_t	hash (const std::string &str) const

void	add (const std::string &)

bool	readWord (std::istream &, std::string &) const

void	readFromFile (std::istream &)

std::string	getLabel (int32_t) const

void	save (std::ostream &) const

void	load (std::istream &)

std::vector< int64_t >	getCounts (entry_type) const

int32_t	getLine (std::istream &, std::vector< int32_t > &, std::vector< int32_t > &, std::vector< int32_t > &, std::minstd_rand &) const

int32_t	getLine (std::istream &, std::vector< int32_t > &, std::vector< int32_t > &, std::minstd_rand &) const

void	threshold (int64_t, int64_t)

void	prune (std::vector< int32_t > &)

Static Public Attributes
static const std::string	EOS = "</s>"

static const std::string	BOW = "<"

static const std::string	EOW = ">"

Private Member Functions
int32_t	find (const std::string &) const

void	initTableDiscard ()

void	initNgrams ()

void	addWordNgrams (std::vector< int32_t > &line, const std::vector< int32_t > &hashes, int32_t n) const

Private Attributes
std::shared_ptr< Args >	args_

std::vector< int32_t >	word2int_

std::vector< entry >	words_

std::vector< real >	pdiscard_

int32_t	size_

int32_t	nwords_

int32_t	nlabels_

int64_t	ntokens_

int64_t	pruneidx_size_ = -1

std::unordered_map< int32_t, int32_t >	pruneidx_

Static Private Attributes
static const int32_t	MAX_VOCAB_SIZE = 30000000

static const int32_t	MAX_LINE_SIZE = 1024

Constructor & Destructor Documentation

◆ Dictionary()

fasttext::Dictionary::Dictionary ( std::shared_ptr< Args > args )

explicit

Member Function Documentation

◆ add()

void fasttext::Dictionary::add ( const std::string & w )

◆ addWordNgrams()

void fasttext::Dictionary::addWordNgrams	(	std::vector< int32_t > &	line,
		const std::vector< int32_t > &	hashes,
		int32_t	n
	)		const

private

◆ computeSubwords() [1/2]

void fasttext::Dictionary::computeSubwords	(	const std::string &	word,
		std::vector< int32_t > &	ngrams
	)		const

◆ computeSubwords() [2/2]

void fasttext::Dictionary::computeSubwords	(	const std::string &	word,
		std::vector< int32_t > &	ngrams,
		std::vector< std::string > &	substrings
	)		const

◆ discard()

bool fasttext::Dictionary::discard	(	int32_t	id,
		real	rand
	)		const

◆ find()

int32_t fasttext::Dictionary::find ( const std::string & w ) const

private

◆ getCounts()

std::vector< int64_t > fasttext::Dictionary::getCounts ( entry_type type ) const

◆ getId()

int32_t fasttext::Dictionary::getId ( const std::string & w ) const

◆ getLabel()

std::string fasttext::Dictionary::getLabel ( int32_t lid ) const

◆ getLine() [1/2]

int32_t fasttext::Dictionary::getLine	(	std::istream &	in,
		std::vector< int32_t > &	words,
		std::vector< int32_t > &	word_hashes,
		std::vector< int32_t > &	labels,
		std::minstd_rand &	rng
	)		const

◆ getLine() [2/2]

int32_t fasttext::Dictionary::getLine	(	std::istream &	in,
		std::vector< int32_t > &	words,
		std::vector< int32_t > &	labels,
		std::minstd_rand &	rng
	)		const

◆ getSubwords() [1/3]

const std::vector< int32_t > & fasttext::Dictionary::getSubwords ( int32_t i ) const

◆ getSubwords() [2/3]

const std::vector< int32_t > fasttext::Dictionary::getSubwords ( const std::string & word ) const

◆ getSubwords() [3/3]

void fasttext::Dictionary::getSubwords	(	const std::string &	word,
		std::vector< int32_t > &	ngrams,
		std::vector< std::string > &	substrings
	)		const

◆ getType() [1/2]

entry_type fasttext::Dictionary::getType ( int32_t id ) const

◆ getType() [2/2]

entry_type fasttext::Dictionary::getType ( const std::string & w ) const

◆ getWord()

std::string fasttext::Dictionary::getWord ( int32_t id ) const

◆ hash()

uint32_t fasttext::Dictionary::hash ( const std::string & str ) const

◆ initNgrams()

void fasttext::Dictionary::initNgrams ( )

private

◆ initTableDiscard()

void fasttext::Dictionary::initTableDiscard ( )

private

◆ load()

void fasttext::Dictionary::load ( std::istream & in )

◆ nlabels()

int32_t fasttext::Dictionary::nlabels ( ) const

◆ ntokens()

int64_t fasttext::Dictionary::ntokens ( ) const

◆ nwords()

int32_t fasttext::Dictionary::nwords ( ) const

◆ prune()

void fasttext::Dictionary::prune ( std::vector< int32_t > & idx )

◆ readFromFile()

void fasttext::Dictionary::readFromFile ( std::istream & in )

◆ readWord()

bool fasttext::Dictionary::readWord	(	std::istream &	in,
		std::string &	word
	)		const

◆ save()

void fasttext::Dictionary::save ( std::ostream & out ) const

◆ threshold()

void fasttext::Dictionary::threshold	(	int64_t	t,
		int64_t	tl
	)

Member Data Documentation

◆ args_

std::shared_ptr<Args> fasttext::Dictionary::args_

private

◆ BOW

const std::string fasttext::Dictionary::BOW = "<"

static

◆ EOS

const std::string fasttext::Dictionary::EOS = "</s>"

static

◆ EOW

const std::string fasttext::Dictionary::EOW = ">"

static

◆ MAX_LINE_SIZE

const int32_t fasttext::Dictionary::MAX_LINE_SIZE = 1024

staticprivate

◆ MAX_VOCAB_SIZE

const int32_t fasttext::Dictionary::MAX_VOCAB_SIZE = 30000000

staticprivate

◆ nlabels_

int32_t fasttext::Dictionary::nlabels_

private

◆ ntokens_

int64_t fasttext::Dictionary::ntokens_

private

◆ nwords_

int32_t fasttext::Dictionary::nwords_

private

◆ pdiscard_

std::vector<real> fasttext::Dictionary::pdiscard_

private

◆ pruneidx_

std::unordered_map<int32_t, int32_t> fasttext::Dictionary::pruneidx_

private

◆ pruneidx_size_

int64_t fasttext::Dictionary::pruneidx_size_ = -1

private

◆ size_

int32_t fasttext::Dictionary::size_

private

◆ word2int_

std::vector<int32_t> fasttext::Dictionary::word2int_

private

◆ words_

std::vector<entry> fasttext::Dictionary::words_

private

The documentation for this class was generated from the following files:

/data/users/cpuhrsch/fbsource/fbcode/deeplearning/fastText/src/dictionary.h
/data/users/cpuhrsch/fbsource/fbcode/deeplearning/fastText/src/dictionary.cc

Public Member Functions

Static Public Attributes

Private Member Functions

Private Attributes

Static Private Attributes

Constructor & Destructor Documentation

◆ Dictionary()

Member Function Documentation

◆ add()

◆ addWordNgrams()

◆ computeSubwords() [1/2]

◆ computeSubwords() [2/2]

◆ discard()

◆ find()

◆ getCounts()

◆ getId()

◆ getLabel()

◆ getLine() [1/2]

◆ getLine() [2/2]

◆ getSubwords() [1/3]

◆ getSubwords() [2/3]

◆ getSubwords() [3/3]

◆ getType() [1/2]

◆ getType() [2/2]

◆ getWord()

◆ hash()

◆ initNgrams()

◆ initTableDiscard()

◆ load()

◆ nlabels()

◆ ntokens()

◆ nwords()

◆ prune()

◆ readFromFile()

◆ readWord()

◆ save()

◆ threshold()

Member Data Documentation

◆ args_

◆ BOW

◆ EOS

◆ EOW

◆ MAX_LINE_SIZE

◆ MAX_VOCAB_SIZE

◆ nlabels_

◆ ntokens_

◆ nwords_

◆ pdiscard_

◆ pruneidx_

◆ pruneidx_size_

◆ size_

◆ word2int_

◆ words_