com.publiccms.common.tools

类 SmartcnDictUtils

java.lang.Object

com.publiccms.common.tools.SmartcnDictUtils

public class SmartcnDictUtils
extends java.lang.Object

原始dict文件工具类 https://github.com/Kerwin23/smartcn-dict

作者:

Kerwin

字段概要

字段

限定符和类型	字段和说明
static int	CHAR_NUM_IN_FILE
private static java.lang.String	CHARSET_GB2312
static java.util.Map<java.lang.String,java.lang.Integer>	defaultDelimiterFreqsMap
static int	GB2312_CHAR_NUM
static int	GB2312_FIRST_CHAR
static java.lang.String	TYPE_BIGRAM
static java.lang.String	TYPE_CORE

构造器概要

构造器

构造器和说明
SmartcnDictUtils()

方法概要

限定符和类型	方法和说明
static void	create(java.lang.String filePath, java.lang.String type, java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> charTFsMap, java.util.Map<java.lang.String,java.lang.Integer> delimiterFreqsMap) 创建词库文件
private static java.lang.String	getCCByGB2312Id(int ccid)
private static short	getGB2312Id(char ch)
private static byte[]	intToLEBytes(int i)
private static void	merge(java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> tfsMap, java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> tempTfsMap, java.util.Set<java.lang.String> keys)
static void	mergeTFsMap(java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> tfsMap, java.util.Map<java.lang.String,java.lang.Integer> source) 增加新的分词
static java.lang.String	readCnTerm(java.lang.String src)
private static void	readCoreMemDelimeter(java.lang.String cc, char[][] termsArray, int[] freqArray, java.util.Map<java.lang.String,java.lang.Integer> delimiterFreqsMap)
private static void	readCoreMemFromArrays(char[][][] wordItemCharArrayTable, int[][] wordItemFrequencyTable, java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> charTermFreqsMap, java.util.Map<java.lang.String,java.lang.Integer> delimiterFreqsMap)
private static void	readCoreMemTerms(java.lang.String cc, char[][] termsArray, int[] freqArray, java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> charTermFreqsMap)
static void	readFromCoreMem(java.io.InputStream inputStream, java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> charTermFreqsMap, java.util.Map<java.lang.String,java.lang.Integer> delimiterFreqsMap) 读取已有词库mem文件
static void	skipWord(java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> tfsMap, java.util.List<java.lang.String> skipWordList) 删除分词
private static java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>>	transSource(java.util.Map<java.lang.String,java.lang.Integer> source)
private static void	writeDelimiters(java.io.OutputStream oStream, java.util.Map<java.lang.String,java.lang.Integer> delimiterFreqsMap)
private static void	writeEmpty(java.io.OutputStream oStream)
private static void	writeInt(java.io.OutputStream oStream, int i)
private static void	writeTFs(java.io.OutputStream oStream, java.util.Map<java.lang.String,java.lang.Integer> tfs)

从类继承的方法 java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

字段详细资料

CHARSET_GB2312

private static final java.lang.String CHARSET_GB2312

另请参阅:

常量字段值

GB2312_FIRST_CHAR

public static final int GB2312_FIRST_CHAR

另请参阅:

常量字段值

GB2312_CHAR_NUM

public static final int GB2312_CHAR_NUM

另请参阅:

常量字段值

CHAR_NUM_IN_FILE

public static final int CHAR_NUM_IN_FILE

另请参阅:

常量字段值

TYPE_CORE

public static final java.lang.String TYPE_CORE

另请参阅:

常量字段值

TYPE_BIGRAM

public static final java.lang.String TYPE_BIGRAM

另请参阅:

常量字段值

defaultDelimiterFreqsMap

public static final java.util.Map<java.lang.String,java.lang.Integer> defaultDelimiterFreqsMap

构造器详细资料

SmartcnDictUtils

public SmartcnDictUtils()

方法详细资料

readCnTerm

public static java.lang.String readCnTerm(java.lang.String src)

readFromCoreMem

public static void readFromCoreMem(java.io.InputStream inputStream,
                                   java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> charTermFreqsMap,
                                   java.util.Map<java.lang.String,java.lang.Integer> delimiterFreqsMap)
                            throws java.io.IOException,
                                   java.lang.ClassNotFoundException

读取已有词库mem文件

参数:

inputStream -

charTermFreqsMap -

delimiterFreqsMap -

抛出:

java.io.IOException

java.lang.ClassNotFoundException

skipWord

public static void skipWord(java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> tfsMap,
                            java.util.List<java.lang.String> skipWordList)

删除分词

参数:

tfsMap -

skipWordList -

mergeTFsMap

public static void mergeTFsMap(java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> tfsMap,
                               java.util.Map<java.lang.String,java.lang.Integer> source)

增加新的分词

参数:

tfsMap -

source -

create

public static void create(java.lang.String filePath,
                          java.lang.String type,
                          java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> charTFsMap,
                          java.util.Map<java.lang.String,java.lang.Integer> delimiterFreqsMap)

创建词库文件

参数:

filePath -

type -

charTFsMap -

delimiterFreqsMap -

transSource

private static java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> transSource(java.util.Map<java.lang.String,java.lang.Integer> source)

readCoreMemFromArrays

private static void readCoreMemFromArrays(char[][][] wordItemCharArrayTable,
                                          int[][] wordItemFrequencyTable,
                                          java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> charTermFreqsMap,
                                          java.util.Map<java.lang.String,java.lang.Integer> delimiterFreqsMap)

readCoreMemTerms

private static void readCoreMemTerms(java.lang.String cc,
                                     char[][] termsArray,
                                     int[] freqArray,
                                     java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> charTermFreqsMap)

readCoreMemDelimeter

private static void readCoreMemDelimeter(java.lang.String cc,
                                         char[][] termsArray,
                                         int[] freqArray,
                                         java.util.Map<java.lang.String,java.lang.Integer> delimiterFreqsMap)

merge

private static void merge(java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> tfsMap,
                          java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Integer>> tempTfsMap,
                          java.util.Set<java.lang.String> keys)

writeDelimiters

private static void writeDelimiters(java.io.OutputStream oStream,
                                    java.util.Map<java.lang.String,java.lang.Integer> delimiterFreqsMap)
                             throws java.lang.Exception

抛出:

java.lang.Exception

writeTFs

private static void writeTFs(java.io.OutputStream oStream,
                             java.util.Map<java.lang.String,java.lang.Integer> tfs)
                      throws java.lang.Exception

抛出:

java.lang.Exception

writeEmpty

private static void writeEmpty(java.io.OutputStream oStream)
                        throws java.lang.Exception

抛出:

java.lang.Exception

writeInt

private static void writeInt(java.io.OutputStream oStream,
                             int i)
                      throws java.lang.Exception

抛出:

java.lang.Exception

intToLEBytes

private static byte[] intToLEBytes(int i)

getCCByGB2312Id

private static java.lang.String getCCByGB2312Id(int ccid)

getGB2312Id

private static short getGB2312Id(char ch)