diff --git a/.classpath b/.classpath
index 10ae32917..4a78265c3 100644
--- a/.classpath
+++ b/.classpath
@@ -15,11 +15,12 @@
+
+
-
-
+
diff --git a/LICENSE.txt b/LICENSE.txt
index a4d044a9e..f6144133f 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2010, Metaweb Technologies, Inc. All rights reserved.
+ * Copyright (c) 2010 Metaweb Technologies, Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -36,9 +36,9 @@ See the 'licenses' directory for a list of the licenses for the libraries we dep
ordered here by license:
licenses/apache2.0.LICENSE.txt
- ant (package org.apache.tools.tar)
- bzip2 (package org.apache.tools.bzip2)
calendar-parser (package com.metaweb.gridworks.expr.util)
+ ant-tools
+ bzip2
commons-lang
commons-codec
jdatapath
@@ -58,7 +58,7 @@ licenses/dom4j.LICENSE.txt (BSD family)
dom4j
licenses/simile.LICENSE.txt (BSD family)
- vicino (package edu.mit.simile.vicino)
+ vicino
licenses/arithcode.LICENSE.txt (BSD family)
arithcode
diff --git a/lib-src/ant-tools-1.8.0-sources.jar b/lib-src/ant-tools-1.8.0-sources.jar
new file mode 100644
index 000000000..bc3be38e3
Binary files /dev/null and b/lib-src/ant-tools-1.8.0-sources.jar differ
diff --git a/lib-src/vicino-1.1-sources.jar b/lib-src/vicino-1.1-sources.jar
new file mode 100644
index 000000000..9960752e0
Binary files /dev/null and b/lib-src/vicino-1.1-sources.jar differ
diff --git a/lib/ant-tools-1.8.0.jar b/lib/ant-tools-1.8.0.jar
new file mode 100644
index 000000000..4006b6682
Binary files /dev/null and b/lib/ant-tools-1.8.0.jar differ
diff --git a/lib/apache-tools-tar.jar b/lib/apache-tools-tar.jar
deleted file mode 100644
index fecb83825..000000000
Binary files a/lib/apache-tools-tar.jar and /dev/null differ
diff --git a/lib/vicino-1.1.jar b/lib/vicino-1.1.jar
new file mode 100644
index 000000000..7752a5df7
Binary files /dev/null and b/lib/vicino-1.1.jar differ
diff --git a/src/main/java/com/metaweb/gridworks/Gridworks.java b/src/main/java/com/metaweb/gridworks/Gridworks.java
index c21ad618e..395222092 100644
--- a/src/main/java/com/metaweb/gridworks/Gridworks.java
+++ b/src/main/java/com/metaweb/gridworks/Gridworks.java
@@ -37,6 +37,7 @@ import com.metaweb.util.threads.ThreadPoolExecutorAdapter;
public class Gridworks {
static private String version;
+ static private File tempDir;
private static Logger root = Logger.getRootLogger();
private static Logger logger = Logger.getLogger("com.metaweb.gridworks");
@@ -65,6 +66,10 @@ public class Gridworks {
return version;
}
+ public static File getTempFile(String name) {
+ return new File(tempDir, name);
+ }
+
public static void main(String[] args) throws Exception {
// tell jetty to use SLF4J for logging instead of its own stuff
@@ -84,6 +89,9 @@ public class Gridworks {
jetty_logger.setLevel(Level.WARN);
version = Configurations.get("gridworks.version","trunk");
+
+ tempDir = new File(Configurations.get("gridworks.temp","temp"));
+ if (!tempDir.exists()) tempDir.mkdirs();
Gridworks gridworks = new Gridworks();
diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java
index 8ba4cac25..fe6793674 100644
--- a/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java
+++ b/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java
@@ -1,6 +1,9 @@
package com.metaweb.gridworks.commands.edit;
import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -9,12 +12,24 @@ import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLConnection;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Properties;
+import java.util.Map.Entry;
+import java.util.zip.GZIPInputStream;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.tools.bzip2.CBZip2InputStream;
+import org.apache.tools.tar.TarEntry;
+import org.apache.tools.tar.TarInputStream;
+
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import com.metaweb.gridworks.Gridworks;
@@ -91,7 +106,9 @@ public class CreateProjectCommand extends Command {
while ((part = parser.readNextPart()) != null) {
if (part.isFile()) {
- internalImportFilePart((FilePart) part, project, options);
+
+ FilePart filePart = (FilePart) part;
+ internalImportFile(project, options, filePart.getFileName(), filePart.getInputStream());
} else if (part.isParam()) {
ParamPart paramPart = (ParamPart) part;
@@ -118,15 +135,120 @@ public class CreateProjectCommand extends Command {
}
}
- protected void internalImportFilePart(
- FilePart filePart,
+ protected void internalImportFile(
Project project,
- Properties options
+ Properties options,
+ String fileName,
+ InputStream inputStream
) throws Exception {
-
- Importer importer = guessImporter(options, null, filePart.getFileName());
-
- internalInvokeImporter(project, importer, options, filePart.getInputStream(), null);
+
+ if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tar.bz2")) {
+ // first, save the file on disk, since we need two passes and we might
+ // not have enough memory to keep it all in there
+ File file = save(inputStream);
+
+ // in the first pass, gather statistics about what files are in there
+ // unfortunately, we have to rely on files extensions, which is horrible but
+ // better than nothing
+ BufferedInputStream stream = new BufferedInputStream(new FileInputStream(file));
+ InputStream is = (fileName.endsWith(".tar.gz")) ? new GZIPInputStream(stream): new CBZip2InputStream(stream);
+ TarInputStream tis = new TarInputStream(is);
+ HashMap ext_map = new HashMap();
+ while (true) {
+ TarEntry entry = tis.getNextEntry();
+ if (entry == null) break;
+ if (!entry.isDirectory()) {
+ String name = entry.getName();
+ String ext = getExtension(name)[1];
+ if (ext_map.containsKey(ext)) {
+ ext_map.put(ext, ext_map.get(ext) + 1);
+ } else {
+ ext_map.put(ext, 1);
+ }
+ }
+ }
+ stream.close();
+
+ // sort extensions by how often they appear
+ List> values = new ArrayList>(ext_map.entrySet());
+ Collections.sort(values, new ValuesComparator());
+
+ if (values.size() == 0) {
+ throw new RuntimeException("The archive contains no files.");
+ }
+
+ // this will contain the set of extensions we'll load from the archive
+ HashSet exts = new HashSet();
+
+ // find the extension that is most frequent or those who share the highest frequency value
+ Entry most_frequent = values.get(0);
+ Entry second_most_frequent = values.get(1);
+ if (most_frequent.getValue() > second_most_frequent.getValue()) { // we have a winner
+ exts.add(most_frequent.getKey());
+ } else { // multiple extensions have the same frequency
+ int winning_frequency = most_frequent.getValue();
+ for (Entry e : values) {
+ if (e.getValue() == winning_frequency) {
+ exts.add(e.getKey());
+ }
+ }
+ }
+ Gridworks.log("Most frequent extensions: " + exts.toString());
+
+
+ } else if (fileName.endsWith(".zip")) {
+
+ } else if (fileName.endsWith(".gz")) {
+ String[] frags = getExtension(fileName);
+ internalImportFile(project, options, frags[0], new GZIPInputStream(inputStream));
+ } else if (fileName.endsWith(".bz2")) {
+ String[] frags = getExtension(fileName);
+ internalImportFile(project, options, frags[0], new CBZip2InputStream(inputStream));
+ } else {
+ load(project, options, fileName, inputStream);
+ }
+ }
+
+ public class ValuesComparator implements Comparator> {
+ public int compare(Entry o1, Entry o2) {
+ return o2.getValue() - o1.getValue();
+ }
+ }
+
+ private void load(Project project, Properties options, String fileName, InputStream inputStream) throws Exception {
+ Importer importer = guessImporter(options, null, fileName);
+ internalInvokeImporter(project, importer, options, inputStream, null);
+ inputStream.close();
+ }
+
+ private File save(InputStream is) throws IOException {
+ File temp = Gridworks.getTempFile(Long.toString(System.currentTimeMillis()));
+ temp.deleteOnExit();
+ copy(is,temp);
+ is.close();
+ return temp;
+ }
+
+ private String[] getExtension(String filename) {
+ String[] result = new String[2];
+ int ext_index = filename.lastIndexOf(".");
+ result[0] = (ext_index == -1) ? filename : filename.substring(0,ext_index);
+ result[1] = (ext_index == -1) ? "" : filename.substring(ext_index + 1);
+ return result;
+ }
+
+ private static long copy(InputStream input, File file) throws IOException {
+ FileOutputStream output = new FileOutputStream(file);
+ byte[] buffer = new byte[4 * 1024];
+ long count = 0;
+ int n = 0;
+ while (-1 != (n = input.read(buffer))) {
+ output.write(buffer, 0, n);
+ count += n;
+ }
+ output.close();
+ input.close();
+ return count;
}
protected void internalImportURL(
@@ -237,17 +359,9 @@ public class CreateProjectCommand extends Command {
new InputStreamReader(inputStream);
}
- try {
- importer.read(reader, project, options, skip, limit);
- } finally {
- reader.close();
- }
+ importer.read(reader, project, options, skip, limit);
} else {
- try {
- importer.read(inputStream, project, options, skip, limit);
- } finally {
- inputStream.close();
- }
+ importer.read(inputStream, project, options, skip, limit);
}
}
@@ -295,6 +409,12 @@ public class CreateProjectCommand extends Command {
return new ExcelImporter(false);
} else if("application/x-xls".equals(contentType)) {
return new ExcelImporter(true);
+ } else if("application/xml".equals(contentType) ||
+ "text/xml".equals(contentType) ||
+ "application/rss+xml".equals(contentType) ||
+ "application/atom+xml".equals(contentType) ||
+ "application/rdf+xml".equals(contentType)) {
+ return new XmlImporter();
}
} else if (fileName != null) {
fileName = fileName.toLowerCase();
diff --git a/src/main/java/edu/mit/simile/vicino/Cluster.java b/src/main/java/edu/mit/simile/vicino/Cluster.java
deleted file mode 100644
index dd5b9c327..000000000
--- a/src/main/java/edu/mit/simile/vicino/Cluster.java
+++ /dev/null
@@ -1,79 +0,0 @@
-package edu.mit.simile.vicino;
-
-import java.io.Serializable;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import edu.mit.simile.vicino.clustering.Clusterer;
-import edu.mit.simile.vicino.clustering.NGramClusterer;
-import edu.mit.simile.vicino.clustering.VPTreeClusterer;
-import edu.mit.simile.vicino.distances.Distance;
-
-public class Cluster extends Operator {
-
- public static void main(String[] args) throws Exception {
- (new Cluster()).init(args);
- }
-
- public void init(String[] args) throws Exception {
- Distance distance = getDistance(args[0]);
- List strings = getStrings(args[1]);
- double radius = Double.parseDouble(args[2]);
- int blocking_size = Integer.parseInt(args[3]);
-
- long vptree_start = System.currentTimeMillis();
- Clusterer vptree_clusterer = new VPTreeClusterer(distance);
- for (String s: strings) {
- vptree_clusterer.populate(s);
- }
- List> vptree_clusters = vptree_clusterer.getClusters(radius);
- long vptree_elapsed = System.currentTimeMillis() - vptree_start;
- int vptree_distances = distance.getCount();
- distance.resetCounter();
-
- long ngram_start = System.currentTimeMillis();
- Clusterer ngram_clusterer = new NGramClusterer(distance,blocking_size);
- for (String s: strings) {
- ngram_clusterer.populate(s);
- }
- List> ngram_clusters = ngram_clusterer.getClusters(radius);
- long ngram_elapsed = System.currentTimeMillis() - ngram_start;
- int ngram_distances = distance.getCount();
- distance.resetCounter();
-
- log("VPTree found " + vptree_clusters.size() + " in " + vptree_elapsed + " ms with " + vptree_distances + " distances\n");
- log("NGram found " + ngram_clusters.size() + " in " + ngram_elapsed + " ms with " + ngram_distances + " distances\n");
-
- if (vptree_clusters.size() > ngram_clusters.size()) {
- log("VPTree clusterer found these clusters the other method couldn't: ");
- diff(vptree_clusters,ngram_clusters);
- } else if (ngram_clusters.size() > vptree_clusters.size()) {
- log("NGram clusterer found these clusters the other method couldn't: ");
- diff(ngram_clusters,vptree_clusters);
- }
-
- System.exit(0);
- }
-
- private void diff(List> more, List> base) {
- Set> holder = new HashSet>(base.size());
-
- for (Set s : base) {
- holder.add(s);
- }
-
- for (Set s : more) {
- if (!holder.contains(s)) {
- printCluster(s);
- }
- }
- }
-
- private void printCluster(Set cluster) {
- for (Serializable s : cluster) {
- log(s.toString());
- }
- log("");
- }
-}
diff --git a/src/main/java/edu/mit/simile/vicino/Distributor.java b/src/main/java/edu/mit/simile/vicino/Distributor.java
deleted file mode 100644
index b9064817d..000000000
--- a/src/main/java/edu/mit/simile/vicino/Distributor.java
+++ /dev/null
@@ -1,61 +0,0 @@
-package edu.mit.simile.vicino;
-
-import java.util.List;
-
-import edu.mit.simile.vicino.distances.Distance;
-
-public class Distributor extends Operator {
-
- private static final int COLUMNS = 70;
- private static final char CHAR = '=';
-
- public static void main(String[] args) throws Exception {
-
- Distance d = getDistance(args[0]);
-
- List strings = getStrings(args[1]);
-
- int buckets = Integer.parseInt(args[2]);
-
- long start = System.currentTimeMillis();
- int[] values = new int[buckets];
-
- int size = strings.size();
- for (int i = 0; i < size; i++) {
- String x = (String) strings.get((int) (Math.random() * size));
- String y = (String) strings.get((int) (Math.random() * size));
- int dist = (int) (buckets * d.d(x, y));
- values[dist]++;
- System.out.print(".");
- }
- System.out.println();
-
- long stop = System.currentTimeMillis();
- float m = ((float) (stop - start)) / (float) size;
-
- int maxValue = 0;
- for (int i = 0; i < buckets; i++) {
- if (values[i] > maxValue) {
- maxValue = values[i];
- }
- }
-
- System.out
- .println("+-------------------------------------------------------------------");
- for (int i = 0; i < buckets; i++) {
- System.out.println("|" + bar(COLUMNS * values[i] / maxValue));
- }
- System.out
- .println("+-------------------------------------------------------------------");
-
- System.out.println("\n Each distance calculation took: " + m + " millis");
- }
-
- static private String bar(int value) {
- StringBuffer b = new StringBuffer(value);
- for (int i = 0; i < value; i++) {
- b.append(CHAR);
- }
- return b.toString();
- }
-}
diff --git a/src/main/java/edu/mit/simile/vicino/Meter.java b/src/main/java/edu/mit/simile/vicino/Meter.java
deleted file mode 100644
index b4fb8cf69..000000000
--- a/src/main/java/edu/mit/simile/vicino/Meter.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package edu.mit.simile.vicino;
-
-import edu.mit.simile.vicino.distances.Distance;
-
-public class Meter extends Operator {
-
- public static void main(String[] args) throws Exception {
- Distance d = getDistance(args[0]);
- System.out.println(args[1] + " <- " + d.d(args[1], args[2]) + " -> " + args[2]);
- }
-
-}
diff --git a/src/main/java/edu/mit/simile/vicino/NGramTokenizer.java b/src/main/java/edu/mit/simile/vicino/NGramTokenizer.java
deleted file mode 100644
index 6d4891e82..000000000
--- a/src/main/java/edu/mit/simile/vicino/NGramTokenizer.java
+++ /dev/null
@@ -1,94 +0,0 @@
-package edu.mit.simile.vicino;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.regex.Pattern;
-
-import com.wcohen.ss.api.Token;
-import com.wcohen.ss.api.Tokenizer;
-
-public class NGramTokenizer implements Tokenizer {
-
- private int ngram_size;
-
- public NGramTokenizer(int ngram_size) {
- this.ngram_size = ngram_size;
- }
-
- public Token[] tokenize(String str) {
- str = normalize(str);
- List tokens = new ArrayList();
- for (int i = 0; i < str.length(); i++) {
- int index = i + ngram_size;
- if (index <= str.length()) {
- tokens.add(intern(str.substring(i,index)));
- }
- }
- return (Token[]) tokens.toArray(new BasicToken[tokens.size()]);
- }
-
- static final Pattern extra = Pattern.compile("\\p{Cntrl}|\\p{Punct}");
- static final Pattern whitespace = Pattern.compile("\\p{Space}+");
-
- private String normalize(String s) {
- s = s.trim();
- s = extra.matcher(s).replaceAll("");
- s = whitespace.matcher(s).replaceAll(" ");
- s = s.toLowerCase();
- return s.intern();
- }
-
- private int nextId = 0;
- private Map tokMap = new TreeMap();
-
- public Token intern(String s) {
- s = s.toLowerCase().intern();
- Token tok = tokMap.get(s);
- if (tok == null) {
- tok = new BasicToken(++nextId, s);
- tokMap.put(s, tok);
- }
- return tok;
- }
-
- public Iterator tokenIterator() {
- return tokMap.values().iterator();
- }
-
- public int maxTokenIndex() {
- return nextId;
- }
-
- public class BasicToken implements Token, Comparable {
- private final int index;
- private final String value;
-
- BasicToken(int index, String value) {
- this.index = index;
- this.value = value;
- }
-
- public String getValue() {
- return value;
- }
-
- public int getIndex() {
- return index;
- }
-
- public int compareTo(Token t) {
- return index - t.getIndex();
- }
-
- public int hashCode() {
- return value.hashCode();
- }
-
- public String toString() {
- return "[token#" + getIndex() + ":" + getValue() + "]";
- }
- }
-}
diff --git a/src/main/java/edu/mit/simile/vicino/Operator.java b/src/main/java/edu/mit/simile/vicino/Operator.java
deleted file mode 100644
index b064860cf..000000000
--- a/src/main/java/edu/mit/simile/vicino/Operator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package edu.mit.simile.vicino;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.mit.simile.vicino.distances.Distance;
-
-public class Operator {
-
- static void log(String msg) {
- System.out.println(msg);
- }
-
- static Distance getDistance(String distance) throws Exception {
- return (Distance) Class.forName("edu.mit.simile.vicino.distances." + distance + "Distance").newInstance();
- }
-
- static List getStrings(String fileName) throws IOException {
- List strings = new ArrayList();
-
- File file = new File(fileName);
- if (file.isDirectory()) {
- File[] files = file.listFiles();
- for (File f : files) {
- getStrings(f, strings);
- }
- } else {
- getStrings(file, strings);
- }
-
- return strings;
- }
-
- static void getStrings(File file, List strings) throws IOException {
- BufferedReader input = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
- String line;
- while ((line = input.readLine()) != null) {
- strings.add(line.trim().intern());
- }
- input.close();
- }
-}
diff --git a/src/main/java/edu/mit/simile/vicino/Seeker.java b/src/main/java/edu/mit/simile/vicino/Seeker.java
deleted file mode 100644
index 7993e8bc9..000000000
--- a/src/main/java/edu/mit/simile/vicino/Seeker.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package edu.mit.simile.vicino;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.io.Serializable;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import edu.mit.simile.vicino.distances.Distance;
-import edu.mit.simile.vicino.vptree.VPTree;
-import edu.mit.simile.vicino.vptree.VPTreeBuilder;
-import edu.mit.simile.vicino.vptree.VPTreeSeeker;
-
-public class Seeker extends Operator {
-
- public static void main(String[] args) throws Exception {
- Distance d = getDistance(args[0]);
-
- log("Working with distance: " + d);
- List strings = getStrings(args[1]);
- log("Obtained " + strings.size() + " from " + args[1]);
-
- log("Building VPTree...");
- VPTreeBuilder builder = new VPTreeBuilder(d);
- VPTree tree = builder.buildVPTree(strings);
- log("..done");
-
- VPTreeSeeker seeker = new VPTreeSeeker(d, tree);
-
- log("type a string|range then hit return:");
- BufferedReader input = new BufferedReader(new InputStreamReader(System.in));
- String line = null;
- while ((line = input.readLine()) != null) {
- int index = line.indexOf('|');
- String query = line.substring(0, index);
- float range = Float.parseFloat(line.substring(index + 1));
- long start = System.currentTimeMillis();
- Set results = seeker.range(query, range);
- long stop = System.currentTimeMillis();
- Iterator j = results.iterator();
- if (j.hasNext()) {
- while (j.hasNext()) {
- String r = (String) j.next();
- log(" " + r);
- }
- log(" [done in " + (stop - start) + "ms]");
- } else {
- log(" [no results found in " + (stop - start) + "ms]");
- }
- }
- }
-}
diff --git a/src/main/java/edu/mit/simile/vicino/Tester.java b/src/main/java/edu/mit/simile/vicino/Tester.java
deleted file mode 100644
index 8dfc3400d..000000000
--- a/src/main/java/edu/mit/simile/vicino/Tester.java
+++ /dev/null
@@ -1,46 +0,0 @@
-package edu.mit.simile.vicino;
-
-import java.util.List;
-
-import edu.mit.simile.vicino.distances.Distance;
-
-public class Tester extends Operator {
-
- public static void main(String[] args) throws Exception {
- Distance d = getDistance(args[0]);
-
- List strings = getStrings(args[1]);
-
- long start = System.currentTimeMillis();
-
- int size = strings.size();
- for (int i = 0; i < size * size * size; i++) {
- String x = (String) strings.get((int) (Math.random() * size));
- String y = (String) strings.get((int) (Math.random() * size));
- String z = (String) strings.get((int) (Math.random() * size));
- boolean metric = metric(x, y, z, d);
- if (metric) {
- System.out.println("metric");
- } else {
- System.out.println("***** NOT METRIC *****");
- }
- }
-
- long stop = System.currentTimeMillis();
- float m = ((float) (stop - start)) / (float) size;
-
- System.out.println("\n Each metric evaluation took: " + m + " millis");
- }
-
- static boolean metric(String x, String y, String z, Distance d) {
- double dxx = d.d(x, x);
- boolean identity = (dxx == 0.0f);
- double dxy = d.d(x, y);
- double dyx = d.d(y, x);
- boolean simmetrical = (dxy == dyx);
- double dxz = d.d(x, z);
- double dyz = d.d(y, z);
- boolean triangular = (dxz <= dxy + dyz);
- return (identity && simmetrical && triangular);
- }
-}
diff --git a/src/main/java/edu/mit/simile/vicino/clustering/Clusterer.java b/src/main/java/edu/mit/simile/vicino/clustering/Clusterer.java
deleted file mode 100644
index 89b72d8f5..000000000
--- a/src/main/java/edu/mit/simile/vicino/clustering/Clusterer.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package edu.mit.simile.vicino.clustering;
-
-import java.io.Serializable;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Set;
-
-public abstract class Clusterer {
-
- public class SizeComparator implements Comparator> {
- public int compare(Set o1, Set o2) {
- return o2.size() - o1.size();
- }
- }
-
- public abstract void populate(String s);
-
- public abstract List> getClusters(double radius);
-
-}
diff --git a/src/main/java/edu/mit/simile/vicino/clustering/NGramClusterer.java b/src/main/java/edu/mit/simile/vicino/clustering/NGramClusterer.java
deleted file mode 100644
index bbf3dacbf..000000000
--- a/src/main/java/edu/mit/simile/vicino/clustering/NGramClusterer.java
+++ /dev/null
@@ -1,194 +0,0 @@
-package edu.mit.simile.vicino.clustering;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.Map.Entry;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import com.wcohen.ss.api.Token;
-
-import edu.mit.simile.vicino.NGramTokenizer;
-import edu.mit.simile.vicino.distances.Distance;
-
-public class NGramClusterer extends Clusterer {
-
- NGramTokenizer _tokenizer;
- Distance _distance;
-
- Map> blocks = new HashMap>();
-
- public NGramClusterer(Distance d, int blockSize) {
- _tokenizer = new NGramTokenizer(blockSize);
- _distance = d;
- }
-
- public void populate(String s) {
- Token[] tokens = _tokenizer.tokenize(s);
- for (Token t : tokens) {
- String ss = t.getValue();
- Set l = null;
- if (!blocks.containsKey(ss)) {
- l = new TreeSet();
- blocks.put(ss, l);
- } else {
- l = blocks.get(ss);
- }
- l.add(s);
- }
- }
-
- public class BlockEvaluator implements Callable