package se.lth.cs.nlp.opinions; import se.lth.cs.nlp.nlputils.core.*; import se.lth.cs.nlp.nlputils.ml_long.featuresearch.*; import java.io.*; import java.util.Arrays; public class SeqLabelerEvaluator extends Function { private String trainFileName; private int nCVFolds = -1; private String methodName; private String[] methodArgs; private boolean secondOrder; private String outputFeature; public SeqLabelerEvaluator(String[] argv) { trainFileName = argv[0]; nCVFolds = Integer.parseInt(argv[1]); secondOrder = Boolean.parseBoolean(argv[2]); outputFeature = argv[3]; methodName = argv[4]; methodArgs = CollectionUtils.subArray(argv, 5, argv.length - 5); System.out.println("trainFile = " + trainFileName); System.out.println("nCVFolds = " + nCVFolds); System.out.println("secondOrder = " + secondOrder); System.out.println("outputFeature = " + outputFeature); System.out.println("methodName = " + methodName); System.out.println("methodArgs = " + Arrays.toString(methodArgs)); } private static void printFeatureSet(String fileName, String outputFeature, FeatureSet f) throws IOException { PrintWriter pw = new PrintWriter(new FileWriter(fileName)); boolean seen = false; for(String feature: f.fs) { if(seen) pw.println(","); seen = true; pw.print(outputFeature + " " + feature); } for(Pair p: f.ps) { if(seen) pw.println(","); seen = true; pw.print(outputFeature + " " + p.left + " " + p.right); } if(seen) pw.println(); pw.close(); } public Double apply(FeatureSet features) { try { String templateFile = Util.getTempFileName("slev"); printFeatureSet(templateFile, outputFeature, features); String[] argv = new String[6 + methodArgs.length]; argv[0] = "-cv"; argv[1] = trainFileName; argv[2] = templateFile; argv[3] = "" + nCVFolds; argv[4] = "" + secondOrder; argv[5] = methodName; System.arraycopy(methodArgs, 0, argv, 6, methodArgs.length); double[] stats = SeqLabeler.cv(argv); new File(templateFile).delete(); double nGuesses = stats[1]; double nInGold = stats[2]; double propGuessCorrect = stats[3]; double propFoundCorrect = stats[4]; double fSoft; if(nGuesses == 0) fSoft = 0; else { double pSoft = propGuessCorrect / nGuesses; double rSoft = propFoundCorrect / nInGold; if(pSoft == 0 || rSoft == 0) fSoft = 0; else fSoft = 2*pSoft*rSoft / (pSoft + rSoft); } return fSoft; } catch(IOException e) { throw new RuntimeException(e); } } }