
package iitb.cfilt.cpost.crfpp;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.*;

import iitb.cfilt.cpost.*;
import iitb.cfilt.cpost.newstemmer.*;
import iitb.cfilt.cpost.ma.*;
import iitb.cfilt.cpost.vgi.*;

public class FileFormer
{
	private static Stemmer stemmer;
	private static MorphologicalAnalyzer ma;
	private static MorphologicalAnalyzerRuleReader MAR;
	private static VerbGroup7 vg;
		
	public static Vector<String> categories;
	public static Vector<String> tagcats;
	public static Vector<String> distinctTags;
	
	private static boolean NNFlag = true; // It is true when NN-NNP distinction is not required
	private static boolean VGI = true; // It is true when VGI is to be applied. By Nikhilesh 
	/**
	 * This function is used to create the training file required for the CRF++ package.
	 * @param inFile This is the input file with tagged data
	 * @param outFile This file is created for using as a training file for CRF++. It contains multiple columns for creating features.
	 * @param flag This is to set NNFlag (see above) // Nikhilesh //
	 * @param flagVGI This is to set VGIFlag (see above) // Nikhilesh //
	 * 
	 */
		
	public static void CreateFile(String inFile, String outFile, String flag,String flagVGI, String stemmer_output, String morphological, String tags)//,String resultfile)
	{
		try
		{
			/* START : To assist to the Web Interface. It needs Intermediatory outputs as Stemmer and Morph. : By Neha*/
			String morpho_file = outFile + "_morpho";
			String stemmer_file = outFile + "_stemmer";
			
			
			File file_stemmer = new File(stemmer_file);
			if(file_stemmer.exists())
			{
				file_stemmer.delete();
				file_stemmer = new File(stemmer_file);
			}
			UTFWriter ob_stemmer = new UTFWriter(file_stemmer);
						
			File file_morpho = new File(morpho_file);
			if(file_morpho.exists())
			{
				file_morpho.delete();
				file_morpho = new File(morpho_file);
			}
			UTFWriter ob_morpho = new UTFWriter(file_morpho);
			
			file_stemmer.createNewFile();
        	file_morpho.createNewFile();
        	/* End */
			/* Nikhilesh : To read the flags. //  start*/
			if(flag.equalsIgnoreCase("true"))
			{
				NNFlag = true;
			}
			else if(flag.equalsIgnoreCase("false"))
			{
				NNFlag = false;
			}
			if(flagVGI.equalsIgnoreCase("true"))
			{
				VGI = true;
			}
			else if(flagVGI.equalsIgnoreCase("false"))
			{
				VGI = false;
			}
			/* Nikhilesh  //  end*/
			String tagFile = ConfigReader.get("CRF.tagfile");
			
			distinctTags = new Vector<String>();
			/***********Temporary code *****************/
			/*File outfile1 = new File(resultfile);
			if(outfile1.exists())
			{
				outfile1.delete();
				outfile1 = new File(resultfile);
			}
			UTFWriter ob1 = new UTFWriter(outfile1);*/
			/***********Temporary code *****************/
			BufferedReader bf = new BufferedReader(new InputStreamReader(new FileInputStream(tagFile), "UTF8"));
			String line = "";
			while((line = bf.readLine()) != null)
			{
//				 ** Remove multiple spaces /*String[]
				String[] vals = line.split(" ");
				StringBuilder sb = new StringBuilder();
				for (String s : vals) { 
					if (s.trim().length() > 0) {
						sb.append(s).append(" "); 
					} 
				} 
				String new_line = sb.toString().trim();
				line = new_line;
				distinctTags.add(line.trim());
			}
			bf.close();
			
			categories = new Vector<String>();
			tagcats = new Vector<String>();
						
			String tagCatMapFile = ConfigReader.get("HMM.tagCat");
			bf = new BufferedReader(new InputStreamReader(new FileInputStream(tagCatMapFile), "UTF8"));
			line = "";
			while((line = bf.readLine()) != null)
			{
				String[] temp = line.split("-");
				categories.add(temp[1].trim());
				tagcats.add(temp[0].trim());
			}
			bf.close();
			
			File outfile = new File(outFile);
			if(outfile.exists())
			{
				outfile.delete();
				outfile = new File(outFile);
			}
			UTFWriter ob = new UTFWriter(outfile);
			
			float rVaux=0,pVaux=0,nVaux=0;
			float rVm=0,pVm=0,nVm=0;
			bf = new BufferedReader(new InputStreamReader(new FileInputStream(inFile), "UTF8"));
			line = "";
			HashMap<String,StemmedToken> stemmerResultsHash = new HashMap<String, StemmedToken>();
			Vector<String> tokens= new Vector<String>(); 
			while((line = bf.readLine()) != null)
			{
				if(line.length() != 0)
				{
					//int [] vga;
					//vga = vg.doVGI_forSentence(line, true);
					Vector<String> words = new Vector<String>();
					Vector<String> labels = new Vector<String>();
					String[] taggedWords = line.split("\\]");
					
					Vector<MorphologicallyAnalyzedToken> matv = new Vector<MorphologicallyAnalyzedToken>();
					Vector<StemmedToken> stv = new Vector<StemmedToken>();
					String token;
					
					for(int i=0;i<taggedWords.length;i++)
					{
						if(taggedWords[i].contains("_"))
						{
							//System.out.println("word.."+taggedWords[i]+"..done");
							String[] taggedWordComponents = taggedWords[i].split("_\\[");
							if(taggedWordComponents[0].trim()!="")
							 token = taggedWordComponents[0].trim();
							else token=".";
							String temptag = taggedWordComponents[1].trim();
							/***********Temporary code *****************/
							/*if(temptag.equals("NN")){
								ob1.writeUTF(token + " ");
							}*/
							/***********Temporary code *****************/
							if(NNFlag) // It is true when NN-NNP distinction is not required
							{
								if(temptag.equals("NNC") || temptag.equals("NNP") || temptag.equals("NNPC") || temptag.equals("XC"))
									temptag = "NN";
								else if(temptag.equals("JJC"))
									temptag = "JJ";
							}
							else
							{
								if(temptag.equals("NNPC"))
									temptag = "NNP";
								else if(temptag.equals("NNC"))
									temptag = "NN";
								else if(temptag.equals("JJC"))
									temptag = "JJ";
							}
							
							//Each word is stemmed first.
							StemmedToken st;
							if(stemmerResultsHash.keySet().contains(token)){
								st = stemmerResultsHash.get(token);
							}
							else{
								st = stemmer.stem(token);
								stemmerResultsHash.put(token, st);
								/* START : To assist to the Web Interface. It needs Intermediatory outputs as Stemmer and Morph. : By Neha*/
								if(stemmer_output.equals("true") && !token.equals("."))
								{
									ob_stemmer.writeUTF("\nToken = " + st.getToken() + "\n");
									
									//StemmedToken sttemp = stv.elementAt(stv.size()-1);							
									ob_stemmer.writeUTF(st.getStemmedOutputs().toString() + "\n");
								}
								/* End */
							}
							stv.add(st);
							
							//Following loop puts ultimate deletion obtained after stemming in the suffix list.
							
							Iterator<StemmerRuleResult> iterResult = st.getStemmedOutputs().iterator();
							while(iterResult.hasNext())
							{	
								StemmerRuleResult temp = (StemmerRuleResult)iterResult.next();
								if(temp.getSuffixList().indexOf(temp.getUltimateDeletion()) == -1)
									temp.getSuffixList().add(temp.getUltimateDeletion());
							}
							
							//Getting MA output for the word.
							
							MorphologicallyAnalyzedToken mat = ma.analyze(st,MAR);
							//MorphologicallyAnalyzedToken vgit = vgi.identifyVerbGroups(mat);
							
							matv.add(mat);
							
							Vector<StemmerRuleResult> strv = st.getStemmedOutputs();
							HashSet<String> cats = new HashSet<String>();
							
							for(int z=0;z<strv.size();z++)
							{
								cats.add(strv.get(z).getCategory().trim());
							}
							
							words.add(token);
							labels.add(temptag);
						}
					}
					
					Vector<MorphologicallyAnalyzedToken> vgitv;
					int [] vga;
					//if(VGI){
						//vgitv = vgi.identifyRegexVGs(matv);
					//}
					//else{
						vgitv = matv;
					//}
					vga = vg.identifyVerbGroups1(matv);   // Nikhilesh 
					// vga will contain an integer corresponding to each token. where 0 for non verb, 1 for main verb and >1 for verb aux.
					
					//Vector<MorphologicallyAnalyzedToken> vgitv = matv;					
					
					
					for(int i=0;i<vgitv.size();i++)
					{
						Vector<String> Categories = new Vector<String>();
						//System.out.println("Token : " + vgitv.get(i).getToken());
						/* START : To assist to the Web Interface. It needs Intermediatory outputs as Stemmer and Morph. : By Neha*/
						boolean mf = false;
						if(morphological.equals("true")&& !vgitv.get(i).getToken().equals(".") ){
							if (!tokens.contains(vgitv.get(i).getToken())){
								ob_morpho.writeUTF("\nToken = " + vgitv.get(i).getToken() + " + ");
								tokens.add(vgitv.get(i).getToken());
								mf = true;
							}							
						}
						
						/* End */
						for(int j=0;j<vgitv.get(i).getMorphologicalAnalyzerOutputs().size();j++)
						{
							//System.out.println("Analysis String : " + vgitv.get(i).getMorphologicalAnalyzerOutputs().get(j).getAnalysisString());
							/* START : To assist to the Web Interface. It needs Intermediatory outputs as Stemmer and Morph. : By Neha*/
							if(morphological.equals("true")&& !vgitv.get(i).getToken().equals(".") && mf)
								ob_morpho.writeUTF("Analysis String = " + vgitv.get(i).getMorphologicalAnalyzerOutputs().get(j).getAnalysisString() + " + ");
							/* End */
							if(!Categories.contains(vgitv.get(i).getMorphologicalAnalyzerOutputs().get(j).getCategory().trim()))
								Categories.add(vgitv.get(i).getMorphologicalAnalyzerOutputs().get(j).getCategory().trim());
						}
						
						//Getting the ambiguity scheme.
						
						
						Collections.sort(Categories);
						String AS = "";
						
						Vector<Integer> catFlags = new Vector<Integer>();
						for(int j=0;j<categories.size();j++)
							catFlags.add(0);
						if(labels.get(i).equals("VAUX")){
							rVaux++;										
						}
						if(labels.get(i).equals("VM")){
							rVm++;										
						}
						
						if(VGI && vga[i]>0){
							if(vga[i]>1){
								if(vga[i]==100){
									AS = "noun";
									catFlags.set(categories.indexOf("noun"),1);
								}
								else{
									pVaux++;
									if(labels.get(i).equals("VAUX")){
										nVaux++;
									}
									AS = "verb_aux";
									catFlags.set(categories.indexOf("verb_aux"),1);
								}
							}
							else{
								pVm++;
								if(labels.get(i).equals("VM")){
									nVm++;
								}
								AS = "verb";
								catFlags.set(categories.indexOf("verb"),1);
							}
						}
						else{
							for(int j=0;j<Categories.size();j++)
							{
								AS = AS + Categories.get(j) + "*";
								catFlags.set(categories.indexOf(Categories.get(j)),1);
							}
							int l = AS.length();
							AS = AS.substring(0,l-1);
						}
						String cats = "";
						
						for(int j=0;j<categories.size();j++)							
								cats = cats + Integer.toString(catFlags.get(j)) + "\t";						
						
						String suffix = StemmerRuleReader.getSuffix(words.get(i));
						if(suffix == null)
							suffix = "NoSuff";
						
						//*********Suffixes & Prefixes of length 1 to 4
						
						String curr_word = words.get(i);
						int len = curr_word.length();
						String s1="No",s2="No",s3="No",s4="No";
						String p1="No",p2="No",p3="No",p4="No";
						if(len >= 4)
						{
							s1 = curr_word.substring(len-1);
							s2 = curr_word.substring(len-2);
							s3 = curr_word.substring(len-3);
							s4 = curr_word.substring(len-4);
							
							p1 = curr_word.substring(0,1);
							p2 = curr_word.substring(0,2);
							p3 = curr_word.substring(0,3);
							p4 = curr_word.substring(0,4);
							
						}
						else if(len >= 3)
						{
							s1 = curr_word.substring(len-1);
							s2 = curr_word.substring(len-2);
							s3 = curr_word.substring(len-3);
							
							p1 = curr_word.substring(0,1);
							p2 = curr_word.substring(0,2);
							p3 = curr_word.substring(0,3);
						}
						else if(len >= 2)
						{
							s1 = curr_word.substring(len-1);
							s2 = curr_word.substring(len-2);
							
							p1 = curr_word.substring(0,1);
							p2 = curr_word.substring(0,2);
						}
						else if(len >= 1)
						{
							s1 = curr_word.substring(len-1);
							
							p1 = curr_word.substring(0,1);
						}
						
						String combined_suffixes = s1 + "\t" + s2 + "\t" + s3 + "\t" + s4 + "\t";
						
						String combined_prefixes = p1 + "\t" + p2 + "\t" + p3 + "\t" + p4 + "\t";
						
						//**********************************
						//Writing this tab seperated line as one training instance in the output file.
						ob.writeUTF(words.get(i) + "\t" + AS + "\t" + cats + suffix + "\t" + combined_suffixes + Integer.toString(len) + "\t" + combined_prefixes + labels.get(i) + "\n");
						
					}
					ob.writeUTF("\n");
				}
			}
			System.out.println("VM Precision : " + nVm/pVm + ", VM Recall : " + nVm/rVm + ", Vaux Precision : " + nVaux/pVaux + ", Vaux Recall : " + nVaux/rVaux);
			System.out.println("Done!");
			bf.close();
			ob.close();
			ob_morpho.close();
			ob_stemmer.close();			
		}
		catch(Exception e)
		{
			System.out.println(e.toString());
			e.printStackTrace();
		}
		
	}
	
	public static void main(String args[])
	{
		
		ConfigReader.read(args[0]);
		stemmer = new Stemmer();
		MAR = new MorphologicalAnalyzerRuleReader();
		ma = new MorphologicalAnalyzer();
		vg = new VerbGroup7();
		//CreateFile(args[1].trim(),args[2].trim(),args[3].trim(),args[4].trim());//,args[5].trim());
		//if(args.length>5){
			CreateFile(args[1].trim(),args[2].trim(),args[3].trim(),args[4].trim(),args[5].trim(),args[6].trim(),args[7].trim());//,args[5].trim());
		//}
		//else{
			//CreateFile(args[1].trim(),args[2].trim(),args[3].trim(),args[4].trim(),"false","false","false");//,args[5].trim());
		//}
	}
}