自然语言处理之分词算法的实现

本文作者王大锤实现了最长正向匹配算法与最长逆向匹配算法，同时，也顺便写了将两种算法判断取最优的一种的算法。

如下：

package nlp;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.*;
import java.util.Map.Entry;

public class Proj2 {
	
	private String dicPath;
	private Map<String,String> map;
	private String word;
	private static final int MAXLEN=7;
	
	public Proj2(String dicPath){
		this.map=new HashMap<String,String>();
		this.dicPath=dicPath;
	}
	
	public void handle(){
		try {
			BufferedReader br = new BufferedReader(new FileReader(dicPath));
			while(true){
				String line=br.readLine();
				if(line==null)
					break;
				line=new String(line.getBytes(),"utf8");
				String key;
				for(int i=0;i<line.length();i++){
					if(line.charAt(i)==','){
						int pos=i;
						key=line.substring(0,pos);
						String value=line.substring(pos+1,line.length());
						map.put(key,value);
						break;
					}
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		
	}
	
	public String getWord(){
		return word;
	}
	
	public void getInput(String word){
		this.word=word;
	}
	
	public void FMM(String s,String words[]){
		int k=0;
		int index=0;
		int len=s.length();
		int mLen=MAXLEN;
		while(index<s.length()){
			for(mLen=MAXLEN;mLen>1&&index<s.length();mLen--){
				if(index+mLen>s.length()){
					continue;
				}
				if(map.containsKey(s.substring(index,index+mLen))){
					//System.out.print(s.substring(index,index+mLen)+"|");
					words[k++]=s.substring(index,index+mLen);
					index+=mLen;
					break;
				}
			}
			if(mLen==1){
				//System.out.print(s.charAt(index)+"|");
				words[k++]=s.substring(index,index+1);
				index++;
			}
		}
	}//姝ｅ悜姣旇緝
	
	public void BMM(String s,String words[]){
		int k=0;
		int len=s.length();
		int index=len-1;
		int mLen=MAXLEN;
		while(index>=0){
			for(mLen=MAXLEN;mLen>1&&index>=0;mLen--){
				if(index+1-mLen<0)
					continue;
				if(map.containsKey(s.substring(index+1-mLen,index+1))){
					//System.out.print(s.substring(index+1-mLen,index+1));
					words[k++]=s.substring(index+1-mLen,index+1);
					index-=mLen;
					break;
				}
			}
			if(mLen==1){
				if(index>=1){
					//System.out.print(s.charAt(index));
					words[k++]=s.substring(index,index+1);
					index--;
				}else{
					words[k++]=s.substring(index,index+1);
					//System.out.print(s.charAt(index));
					break;
				}
			}
		}
		
	}//閫嗗悜姣旇緝
	
	
	public int compare(String[]fWords,String[]bWords){
		int nWords[]=new int[2];
		int sWords[]=new int[2];
		int tWords[]=new int[2];
		Arrays.fill(nWords,0);
		Arrays.fill(sWords,0);
		Arrays.fill(tWords,0);
		int i;
		for(i=0;i<fWords.length;i++){
			if(fWords.equals(""))
				break;
			if(!map.containsKey(fWords[i])){
				nWords[0]++;
			}
			if(map.containsKey(fWords[i])&&fWords[i].length()==1)
				sWords[0]++;
		}
		tWords[0]=i;
		
		for(i=0;i<bWords.length;i++){
			if(bWords.equals(""))
				break;
			if(!map.containsKey(bWords[i]))
				nWords[1]++;
			if(map.containsKey(bWords[i])&&bWords[i].length()==1)
				sWords[1]++;
		}
		tWords[1]=i;
		
		int total1=nWords[0]+sWords[0]+tWords[0];
		int total2=nWords[1]+sWords[1]+tWords[1];
		if(total1>total2)
			return 1;
		else 
			return 0;
	}
	
	public static void main(String args[]) throws Exception{
		int MAXSIZE=100;
		Proj2 p=new Proj2("d:\\ce.txt");
		p.handle();
		while(true){
			BufferedReader buffer=new BufferedReader(new InputStreamReader(System.in));   
			String str=buffer.readLine();
			p.getInput(str);
			String[] fWords=new String[MAXSIZE];
			Arrays.fill(fWords,"");
			p.FMM(p.getWord(),fWords);
			//if(c==0){
			for(int i=0;i<fWords.length;i++){
				if(fWords[i]!="")
					System.out.print(fWords[i]+"|");
			}
			System.out.println();
			//}else{
			//}
		}
	}
	
	
}