package com.google.refine; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.tools.tar.TarOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.refine.history.HistoryEntryManager; import com.google.refine.model.Project; import com.google.refine.preference.PreferenceStore; import com.google.refine.preference.TopList; /** * ProjectManager is responsible for loading and saving the workspace and projects. * * */ public abstract class ProjectManager { // last n expressions used across all projects static protected final int s_expressionHistoryMax = 100; protected Map _projectsMetadata; protected PreferenceStore _preferenceStore; final static Logger logger = LoggerFactory.getLogger("project_manager"); /** * What caches the joins between projects. */ transient protected InterProjectModel _interProjectModel = new InterProjectModel(); /** * Flags */ transient protected int _busy = 0; // heavy operations like creating or importing projects are going on /** * While each project's metadata is loaded completely at start-up, each project's raw data * is loaded only when the project is accessed by the user. This is because project * metadata is tiny compared to raw project data. This hash map from project ID to project * is more like a last accessed-last out cache. */ transient protected Map _projects; static public ProjectManager singleton; protected ProjectManager(){ _projectsMetadata = new HashMap(); _preferenceStore = new PreferenceStore(); _projects = new HashMap(); preparePreferenceStore(_preferenceStore); } public void dispose() { save(true); // complete save for (Project project : _projects.values()) { if (project != null) { project.dispose(); } } _projects.clear(); _projectsMetadata.clear(); } /** * Registers the project in the memory of the current session * @param project * @param projectMetadata */ public void registerProject(Project project, ProjectMetadata projectMetadata) { synchronized (this) { _projects.put(project.id, project); _projectsMetadata.put(project.id, projectMetadata); } } //----------Load from data store to memory---------------- /** * Load project metadata from data storage * @param projectID * @return */ public abstract boolean loadProjectMetadata(long projectID); /** * Loads a project from the data store into memory * @param id * @return */ protected abstract Project loadProject(long id); //------------Import and Export from Google Refine archive----------------- /** * Import project from a Google Refine archive * @param projectID * @param inputStream * @param gziped * @throws IOException */ public abstract void importProject(long projectID, InputStream inputStream, boolean gziped) throws IOException; /** * Export project to a Google Refine archive * @param projectId * @param tos * @throws IOException */ public abstract void exportProject(long projectId, TarOutputStream tos) throws IOException; //------------Save to record store------------ /** * Saves a project and its metadata to the data store * @param id */ public void ensureProjectSaved(long id) { synchronized(this){ ProjectMetadata metadata = this.getProjectMetadata(id); if (metadata != null) { try { saveMetadata(metadata, id); } catch (Exception e) { e.printStackTrace(); } }//FIXME what should be the behaviour if metadata is null? i.e. not found Project project = getProject(id); if (project != null && metadata != null && metadata.getModified().after(project.getLastSave())) { try { saveProject(project); } catch (Exception e) { e.printStackTrace(); } }//FIXME what should be the behaviour if project is null? i.e. not found or loaded. //FIXME what should happen if the metadata is found, but not the project? or vice versa? } } /** * Save project metadata to the data store * @param metadata * @param projectId * @throws Exception */ protected abstract void saveMetadata(ProjectMetadata metadata, long projectId) throws Exception; /** * Save project to the data store * @param project */ protected abstract void saveProject(Project project); /** * Save workspace and all projects to data store * @param allModified */ public void save(boolean allModified) { if (allModified || _busy == 0) { saveProjects(allModified); saveWorkspace(); } } /** * Saves the workspace to the data store */ protected abstract void saveWorkspace(); /** * A utility class to prioritize projects for saving, depending on how long ago * they have been changed but have not been saved. */ static protected class SaveRecord { final Project project; final long overdue; SaveRecord(Project project, long overdue) { this.project = project; this.overdue = overdue; } } static protected final int s_projectFlushDelay = 1000 * 60 * 60; // 1 hour static protected final int s_quickSaveTimeout = 1000 * 30; // 30 secs /** * Saves all projects to the data store * @param allModified */ protected void saveProjects(boolean allModified) { List records = new ArrayList(); Date startTimeOfSave = new Date(); synchronized (this) { for (long id : _projectsMetadata.keySet()) { ProjectMetadata metadata = getProjectMetadata(id); Project project = _projects.get(id); // don't call getProject() as that will load the project. if (project != null) { boolean hasUnsavedChanges = metadata.getModified().getTime() >= project.getLastSave().getTime(); // We use >= instead of just > to avoid the case where a newly created project // has the same modified and last save times, resulting in the project not getting // saved at all. if (hasUnsavedChanges) { long msecsOverdue = startTimeOfSave.getTime() - project.getLastSave().getTime(); records.add(new SaveRecord(project, msecsOverdue)); } else if (startTimeOfSave.getTime() - project.getLastSave().getTime() > s_projectFlushDelay) { /* * It's been a while since the project was last saved and it hasn't been * modified. We can safely remove it from the cache to save some memory. */ _projects.remove(id).dispose(); } } } } if (records.size() > 0) { Collections.sort(records, new Comparator() { public int compare(SaveRecord o1, SaveRecord o2) { if (o1.overdue < o2.overdue) { return 1; } else if (o1.overdue > o2.overdue) { return -1; } else { return 0; } } }); logger.info(allModified ? "Saving all modified projects ..." : "Saving some modified projects ..." ); for (int i = 0; i < records.size() && (allModified || (new Date().getTime() - startTimeOfSave.getTime() < s_quickSaveTimeout)); i++) { try { saveProject(records.get(i).project); } catch (Exception e) { e.printStackTrace(); } } } } //--------------Get from memory-------------- /** * Gets the InterProjectModel from memory */ public InterProjectModel getInterProjectModel() { return _interProjectModel; } /** * Gets the project metadata from memory * Requires that the metadata has already been loaded from the data store * @param id * @return */ public ProjectMetadata getProjectMetadata(long id) { return _projectsMetadata.get(id); } /** * Gets the project metadata from memory * Requires that the metadata has already been loaded from the data store * @param name * @return */ public ProjectMetadata getProjectMetadata(String name) { for (ProjectMetadata pm : _projectsMetadata.values()) { if (pm.getName().equals(name)) { return pm; } } return null; } /** * Tries to find the project id when given a project name * Requires that all project metadata exists has been loaded to memory from the data store * @param name * The name of the project * @return * The id of the project, or -1 if it cannot be found */ public long getProjectID(String name) { for (Entry entry : _projectsMetadata.entrySet()) { if (entry.getValue().getName().equals(name)) { return entry.getKey(); } } return -1; } /** * Gets all the project Metadata currently held in memory * @return */ public Map getAllProjectMetadata() { return _projectsMetadata; } /** * Gets the required project from the data store * If project does not already exist in memory, it is loaded from the data store * @param id * the id of the project * @return * the project with the matching id, or null if it can't be found */ public Project getProject(long id) { synchronized (this) { if (_projects.containsKey(id)) { return _projects.get(id); } else { Project project = loadProject(id); if (project != null) { _projects.put(id, project); } return project; } } } /** * Gets the preference store * @return */ public PreferenceStore getPreferenceStore() { return _preferenceStore; } /** * Gets all expressions from the preference store * @return */ public List getExpressions() { return ((TopList) _preferenceStore.get("scripting.expressions")).getList(); } /** * The history entry manager deals with changes * @return manager for handling history */ public abstract HistoryEntryManager getHistoryEntryManager(); //-------------remove project----------- /** * Remove the project from the data store * @param project */ public void deleteProject(Project project) { deleteProject(project.id); } /** * Remove project from data store * @param projectID */ public abstract void deleteProject(long projectID); /** * Removes project from memory * @param projectID */ protected void removeProject(long projectID){ if (_projects.containsKey(projectID)) { _projects.remove(projectID).dispose(); } if (_projectsMetadata.containsKey(projectID)) { _projectsMetadata.remove(projectID); } } //--------------Miscellaneous----------- /** * Sets the flag for long running operations * @param busy */ public void setBusy(boolean busy) { synchronized (this) { if (busy) { _busy++; } else { _busy--; } } } /** * Add the latest expression to the preference store * @param s */ public void addLatestExpression(String s) { synchronized (this) { ((TopList) _preferenceStore.get("scripting.expressions")).add(s); } } /** * * @param ps */ static protected void preparePreferenceStore(PreferenceStore ps) { ps.put("scripting.expressions", new TopList(s_expressionHistoryMax)); } }