RandomSec/main/src/com/google/refine/io/FileProjectManager.java

/*

Copyright 2010, Google Inc.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

 * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
 * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
 * Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,           
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY           
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */

package com.google.refine.io;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Properties;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarInputStream;
import org.apache.tools.tar.TarOutputStream;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.refine.ProjectManager;
import com.google.refine.history.HistoryEntryManager;
import com.google.refine.model.Project;
import com.google.refine.model.medadata.DataPackageMetadata;
import com.google.refine.model.medadata.IMetadata;
import com.google.refine.model.medadata.MetadataFormat;
import com.google.refine.model.medadata.ProjectMetadata;
import com.google.refine.preference.TopList;


public class FileProjectManager extends ProjectManager {
    final static protected String PROJECT_DIR_SUFFIX = ".project";

    protected File                       _workspaceDir;

    final static Logger logger = LoggerFactory.getLogger("FileProjectManager");

    static public synchronized void initialize(File dir) {
        if (singleton != null) {
            logger.warn("Overwriting singleton already set: " + singleton);
        }
        logger.info("Using workspace directory: {}", dir.getAbsolutePath());
        singleton = new FileProjectManager(dir);
        // This needs our singleton set, thus the unconventional control flow
        ((FileProjectManager) singleton).recover();
    }

    protected FileProjectManager(File dir) {
        super();
        _workspaceDir = dir;
        if (!_workspaceDir.exists() && !_workspaceDir.mkdirs()) {
            logger.error("Failed to create directory : " + _workspaceDir);
            return;
        }

        load();
    }

    public File getWorkspaceDir() {
        return _workspaceDir;
    }

    static public File getProjectDir(File workspaceDir, long projectID) {
        File dir = new File(workspaceDir, projectID + PROJECT_DIR_SUFFIX);
        if (!dir.exists()) {
            dir.mkdir();
        }
        return dir;
    }

    public File getProjectDir(long projectID) {
        return getProjectDir(_workspaceDir, projectID);
    }

    /**
     * Import an external project that has been received as a .tar file, expanded, and
     * copied into our workspace directory.
     *
     * @param projectID
     */
    @Override
    public boolean loadProjectMetadata(long projectID) {
        synchronized (this) {
            ProjectMetadata metadata = ProjectMetadataUtilities.load(getProjectDir(projectID));
            if (metadata == null) {
                metadata = ProjectMetadataUtilities.recover(getProjectDir(projectID), projectID);
            }
            if (metadata != null) {
                _projectsMetadata.put(projectID, metadata);
                if (_projectsTags == null) {
                    _projectsTags = new HashMap<String, Integer>();
                }
                
                if (metadata != null && metadata.getTags() != null) {
                    for (String tag : metadata.getTags()) {
                      if (_projectsTags.containsKey(tag)) {
                        _projectsTags.put(tag, _projectsTags.get(tag) + 1);
                      } else {
                        _projectsTags.put(tag, 1);
                      }
                    }
                }
                return true;
            } else {
                return false;
            }
        }
    }

    @Override
    public void importProject(long projectID, InputStream inputStream, boolean gziped) throws IOException {
        File destDir = this.getProjectDir(projectID);
        destDir.mkdirs();

        if (gziped) {
            GZIPInputStream gis = new GZIPInputStream(inputStream);
            untar(destDir, gis);
        } else {
            untar(destDir, inputStream);
        }
    }
        
    protected void untar(File destDir, InputStream inputStream) throws IOException {
        TarInputStream tin = new TarInputStream(inputStream);
        TarEntry tarEntry = null;

        while ((tarEntry = tin.getNextEntry()) != null) {
            File destEntry = new File(destDir, tarEntry.getName());
            File parent = destEntry.getParentFile();

            if (!parent.exists()) {
                parent.mkdirs();
            }

            if (tarEntry.isDirectory()) {
                destEntry.mkdirs();
            } else {
                FileOutputStream fout = new FileOutputStream(destEntry);
                try {
                    tin.copyEntryContents(fout);
                } finally {
                    fout.close();
                }
            }
        }
        
        tin.close();
    }

    @Override
    public void exportProject(long projectId, TarOutputStream tos) throws IOException {
        File dir = this.getProjectDir(projectId);
        this.tarDir("", dir, tos);
    }

    protected void tarDir(String relative, File dir, TarOutputStream tos) throws IOException{
        File[] files = dir.listFiles();
        for (File file : files) {
            if (!file.isHidden()) {
                String path = relative + file.getName();

                if (file.isDirectory()) {
                    tarDir(path + File.separator, file, tos);
                } else {
                    TarEntry entry = new TarEntry(path);

                    entry.setMode(TarEntry.DEFAULT_FILE_MODE);
                    entry.setSize(file.length());
                    entry.setModTime(file.lastModified());

                    tos.putNextEntry(entry);

                    copyFile(file, tos);

                    tos.closeEntry();
                }
            }
        }
    }

    protected void copyFile(File file, OutputStream os) throws IOException {
        final int buffersize = 4096;

        FileInputStream fis = new FileInputStream(file);
        try {
            byte[] buf = new byte[buffersize];
            int count;

            while((count = fis.read(buf, 0, buffersize)) != -1) {
                os.write(buf, 0, count);
            }
        } finally {
            fis.close();
        }
    }

    @Override
    public void saveMetadata(IMetadata metadata, long projectId) throws Exception {
        File projectDir = getProjectDir(projectId);
        
        if (metadata.getFormatName() == MetadataFormat.PROJECT_METADATA) {
            Project project = ProjectManager.singleton.getProject(projectId);
            ((ProjectMetadata)metadata).setRowCount(project.rows.size());
            ProjectMetadataUtilities.save(metadata, projectDir);
        } else if (metadata.getFormatName() == MetadataFormat.DATAPACKAGE_METADATA) {
            DataPackageMetadata dp = (DataPackageMetadata)metadata;
            dp.writeToFile(new File(projectDir, DataPackageMetadata.DEFAULT_FILE_NAME));
        }
        
        logger.info("metadata saved in " + metadata.getFormatName());
    }

    @Override
    protected void saveProject(Project project) throws IOException{
        ProjectUtilities.save(project);
    }

    @Override
    public Project loadProject(long id) {
        return ProjectUtilities.load(getProjectDir(id), id);
    }


    /**
     * Save the workspace's data out to file in a safe way: save to a temporary file first
     * and rename it to the real file.
     */
    @Override
    protected void saveWorkspace() {
        synchronized (this) {
            File tempFile = new File(_workspaceDir, "workspace.temp.json");
            try {
                if (!saveToFile(tempFile)) {
                    // If the save wasn't really needed, just keep what we had
                    tempFile.delete();
                    logger.info("Skipping unnecessary workspace save");
                    return;
                }
            } catch (Exception e) {
                e.printStackTrace();

                logger.warn("Failed to save workspace");
                return;
            }

            File file = new File(_workspaceDir, "workspace.json");
            File oldFile = new File(_workspaceDir, "workspace.old.json");

            if (oldFile.exists()) {
                oldFile.delete();
            }
            
            if (file.exists()) {
                file.renameTo(oldFile);
            }

            tempFile.renameTo(file);

            logger.info("Saved workspace");
        }
    }

    protected boolean saveToFile(File file) throws IOException, JSONException {
        FileWriter writer = new FileWriter(file);
        boolean saveWasNeeded = false;
        try {
            JSONWriter jsonWriter = new JSONWriter(writer);
            jsonWriter.object();
            jsonWriter.key("projectIDs");
            jsonWriter.array();
            for (Long id : _projectsMetadata.keySet()) {
                ProjectMetadata metadata = _projectsMetadata.get(id);
                if (metadata != null) {
                    jsonWriter.value(id);
                    if (metadata.isDirty()) {
                        Project project = ProjectManager.singleton.getProject(id);
                        metadata.setRowCount(project.rows.size());
                        ProjectMetadataUtilities.save(metadata, getProjectDir(id));
                        saveWasNeeded = true;
                    }
                }
            }
            jsonWriter.endArray();
            writer.write('\n');

            jsonWriter.key("preferences");
            saveWasNeeded |= _preferenceStore.isDirty();
            _preferenceStore.write(jsonWriter, new Properties());

            jsonWriter.endObject();
        } finally {
            writer.close();
        }
        return saveWasNeeded;
    }

    @Override
    public void deleteProject(long projectID) {
        synchronized (this) {
            removeProject(projectID);

            File dir = getProjectDir(projectID);
            if (dir.exists()) {
                deleteDir(dir);
            }
        }

        saveWorkspace();
    }

    static protected void deleteDir(File dir) {
        for (File file : dir.listFiles()) {
            if (file.isDirectory()) {
                deleteDir(file);
            } else {
                file.delete();
            }
        }
        dir.delete();
    }

    protected void load() {
        if (loadFromFile(new File(_workspaceDir, "workspace.json"))) {
            return;
        }
        if (loadFromFile(new File(_workspaceDir, "workspace.temp.json"))) {
            return;
        }
        if (loadFromFile(new File(_workspaceDir, "workspace.old.json"))) {
            return;
        }
        logger.error("Failed to load workspace from any attempted alternatives.");
    }

    protected boolean loadFromFile(File file) {
        logger.info("Loading workspace: {}", file.getAbsolutePath());

        boolean found = false;

        if (file.exists() || file.canRead()) {
            FileReader reader = null;
            try {
                reader = new FileReader(file);
                JSONTokener tokener = new JSONTokener(reader);
                JSONObject obj = (JSONObject) tokener.nextValue();
                
                // load global preferences firstly
                if (obj.has("preferences") && !obj.isNull("preferences")) {
                    _preferenceStore.load(obj.getJSONObject("preferences"));
                }

                JSONArray a = obj.getJSONArray("projectIDs");
                int count = a.length();
                for (int i = 0; i < count; i++) {
                    long id = a.getLong(i);

                    File projectDir = getProjectDir(id);
                    ProjectMetadata metadata = ProjectMetadataUtilities.load(projectDir);
                    
                    mergeEmptyUserMetadata(metadata);

                    _projectsMetadata.put(id, metadata);
                    
                    if (metadata != null && metadata.getTags() != null) {
                        for (String tag : metadata.getTags()) {
                          if (_projectsTags.containsKey(tag)) {
                            _projectsTags.put(tag, _projectsTags.get(tag) + 1);
                          } else {
                            _projectsTags.put(tag, 1);
                          }
                        }
                    }
                }

                if (obj.has("expressions") && !obj.isNull("expressions")) { // backward compatibility
                    ((TopList) _preferenceStore.get("scripting.expressions"))
                    .load(obj.getJSONArray("expressions"));
                }

                found = true;
            } catch (JSONException e) {
                logger.warn("Error reading file", e);
            } catch (IOException e) {
                logger.warn("Error reading file", e);
            } finally {
                try {
                    if (reader != null) {
                        reader.close();
                    }
                } catch (IOException e) {
                    logger.warn("Exception closing file",e);
                }
            }
        }

        return found;
    }

    protected void recover() {
        boolean recovered = false;
        for (File file : _workspaceDir.listFiles()) {
            if (file.isDirectory() && !file.isHidden()) {
                String dirName = file.getName();
                if (file.getName().endsWith(PROJECT_DIR_SUFFIX)) {
                    String idString = dirName.substring(0, dirName.length() - PROJECT_DIR_SUFFIX.length());
                    long id = -1;
                    try {
                        id = Long.parseLong(idString);
                    } catch (NumberFormatException e) {
                        // ignore
                    }

                    if (id > 0 && !_projectsMetadata.containsKey(id)) {
                        if (loadProjectMetadata(id)) {
                            logger.info("Recovered project named " 
                                    + getProjectMetadata(id).getName()
                                    + " in directory " + dirName);
                            recovered = true;
                        } else {
                            logger.warn("Failed to recover project in directory " + dirName);

                            file.renameTo(new File(file.getParentFile(), dirName + ".corrupted"));
                        }
                    }
                }
            }
        }
        if (recovered) {
            saveWorkspace();
        }
    }

    @Override
    public HistoryEntryManager getHistoryEntryManager(){
        return new FileHistoryEntryManager();
    }
    
    public static void gzipTarToOutputStream(Project project, OutputStream os) throws IOException {
        GZIPOutputStream gos = new GZIPOutputStream(os);
        TarOutputStream tos = new TarOutputStream(gos);
        try {
            ProjectManager.singleton.exportProject(project.id, tos);
        } finally {
            tos.close();
            gos.close();
        }
    }
}