Commit d01ca8b7 authored by mhellka's avatar mhellka
Browse files

Implemented local/dedicated snapshots.

Local snapshots are stored under /snapshots/{name]/ in the same storage object than the archive itself.
Dedicated snapshots are stored in separate storage objects.
parent fb4b8495
......@@ -8,6 +8,7 @@ import de.gwdg.cdstar.runtime.client.CDStarArchive;
import de.gwdg.cdstar.runtime.client.CDStarAttribute;
import de.gwdg.cdstar.runtime.client.CDStarFile;
import de.gwdg.cdstar.runtime.client.CDStarProfile;
import de.gwdg.cdstar.runtime.client.CDStarSnapshot;
import de.gwdg.cdstar.runtime.client.auth.ArchivePermission;
/**
......@@ -32,7 +33,7 @@ public interface ArchiveListener {
default void aclChanged(CDStarArchive archive, CDStarACLEntry entry, Set<ArchivePermission> oldPermissions) {
}
default void propertyChanged(CDStarAttribute property, CDStarFile file, List<String> originalValues) {
default void propertyChanged(CDStarArchive archive, CDStarAttribute property, List<String> originalValues) {
}
default void fileCreated(CDStarFile file) {
......@@ -41,6 +42,9 @@ public interface ArchiveListener {
default void fileNameChanged(CDStarFile file, String oldName) {
}
default void filePropertyChanged(CDStarFile file, CDStarAttribute property, List<String> originalValues) {
}
/**
*
* @param file
......@@ -55,4 +59,10 @@ public interface ArchiveListener {
default void fileRemoved(CDStarFile archiveFile) {
}
default void snapshotCreated(CDStarSnapshot snapshot) {
}
default void snapshotRemoved(CDStarSnapshot snapshot) {
}
}
......@@ -33,7 +33,7 @@ public class ArchiveFileIterable extends AbstractRequestIterator<FileInfo> {
List<String> excludeFilter = new ArrayList<>();
int limit = 1024;
int offset = 0;
int total = -1;
long total = -1;
String order = "name";
boolean reverse;
......@@ -105,9 +105,9 @@ public class ArchiveFileIterable extends AbstractRequestIterator<FileInfo> {
rq.order(order, reverse);
rq.page(offset, limit);
return rq.submit(client).thenApply((FileList list) -> {
offset += list.files.getCount();
total = list.total;
return list.files;
offset += list.getItems().size();
total = list.getTotal();
return list.getItems();
});
}
......
......@@ -42,17 +42,17 @@ public class SnapshotTest extends BaseRestTest {
@Test
public void testListSnapshots() throws Exception {
final String v1 = makeSnapshot(archiveId, "v1");
final String v2 = makeSnapshot(archiveId, "v2");
final String v3 = makeSnapshot(archiveId, "v3");
makeSnapshot(archiveId, "v1");
makeSnapshot(archiveId, "v2");
makeSnapshot(archiveId, "v3");
final SnapshotList snapshots = target("/v3/test/" + archiveId).queryParam("snapshots", "").request()
.get(SnapshotList.class);
assertEquals(3, snapshots.getCount());
assertEquals(3, snapshots.getTotal());
assertEquals("v1", snapshots.getItems().get(0));
assertEquals("v2", snapshots.getItems().get(1));
assertEquals("v3", snapshots.getItems().get(2));
assertEquals("v1", snapshots.getItems().get(0).name);
assertEquals("v2", snapshots.getItems().get(1).name);
assertEquals("v3", snapshots.getItems().get(2).name);
}
@Test
......
......@@ -333,26 +333,12 @@ class ArchiveImpl implements CDStarArchive {
for (final Resource r : sto.getResourcesByPrefix(FileImpl.RESOURCE_PREFIX)) {
String name = r.getName();
name = name.substring(FileImpl.RESOURCE_PREFIX.length(), name.length());
filesCache.add(new FileImpl(this, r, name, false));
filesCache.add(new FileImpl(this, r, name, null));
}
}
return filesCache;
}
Optional<FileImpl> getInternalFileById(String fileId) {
for (FileImpl file : getInternalFileList())
if (fileId.equals(file.getID()))
return Optional.of(file);
return Optional.empty();
}
Optional<FileImpl> getInternalFileByName(String fileName) {
for (FileImpl file : getInternalFileList())
if (fileName.equals(file.getName()))
return Optional.of(file);
return Optional.empty();
}
@Override
public List<CDStarFile> getFiles() {
checkPermission(ArchivePermission.LIST_FILES);
......@@ -402,7 +388,7 @@ class ArchiveImpl implements CDStarArchive {
getInternalFileList(); // Triggers discovery of existing files.
final Resource r = sto.createResource(null);
final FileImpl file = new FileImpl(this, r, null, false);
final FileImpl file = new FileImpl(this, r, null, null);
try {
file.setName(name);
file.setMediaType(mediaType);
......@@ -422,12 +408,13 @@ class ArchiveImpl implements CDStarArchive {
requirePayloadWriteable();
checkPermission(ArchivePermission.CHANGE_FILES);
// Must be loaded before file is removed, or it will complain about references
// Must be loaded before file is removed, or it will complain about
// references
// to missing files.
getMetaNoCheck();
if (filesCache.remove(file)) {
getMetaNoCheck().onFileRemoved(file);
file.resource.remove();
file.getResource().remove();
markContentModified();
}
}
......@@ -483,7 +470,13 @@ class ArchiveImpl implements CDStarArchive {
// requirePayloadWriteable(); <-- Deleting archived archives is allowed.
checkPermission(ArchivePermission.DELETE);
// Remove all snapshots, as some of them might be stored in external
// storage objects.
getSnapshotsInternal().removeAll();
// Kill this storage object
sto.remove();
markContentModified();
listeners.forEach(l -> l.archiveRemoved(this));
}
......@@ -504,7 +497,7 @@ class ArchiveImpl implements CDStarArchive {
private synchronized AttributeCache getMetaNoCheck() {
if (metaCache == null)
metaCache = new AttributeCache(this, sto.getResource(ArchiveImpl.METAFILE), false);
metaCache = new AttributeCache(this, null);
return metaCache;
}
......@@ -562,33 +555,33 @@ class ArchiveImpl implements CDStarArchive {
@Override
public Optional<CDStarSnapshot> getSnapshot(String name) {
return snapshotList.getSnapshot(name);
// checkPermission(ArchivePermission.LOAD);
return getSnapshotsInternal().getSnapshot(name);
}
@Override
public List<CDStarSnapshot> getSnapshots() {
return snapshotList.getSnapshots();
// checkPermission(ArchivePermission.LOAD);
return getSnapshotsInternal().getSnapshots();
}
@Override
public synchronized CDStarSnapshot createSnapshot(String name) throws InvalidSnapshotName {
// TODO: Make this decision more transparent and configurable
boolean localSnapshot = sto.getResourceCount() < 1024;
return createSnapshot(name, localSnapshot);
}
public synchronized CDStarSnapshot createSnapshot(String name, boolean local) throws InvalidSnapshotName {
checkPermission(ArchivePermission.SNAPSHOT);
requireTransactionWriteable();
requirePayloadAvailable();
if (isContentModified())
throw new IllegalStateException("Snapshots can only be created from unmodified content");
if (!name.matches("^[a-z0-9-.]+$"))
throw new InvalidSnapshotName(name, "Unsupported characters");
final StorageObject snapSto = vault.createSTO();
try {
final CDStarSnapshot snap = getSnapshotsInternal().createSnapshot(name, snapSto);
final CDStarSnapshot snap = getSnapshotsInternal().createSnapshot(name, local);
markModified();
return snap;
} catch (InvalidSnapshotName | IOException | RuntimeException e) {
snapSto.remove();
if (e instanceof InvalidSnapshotName)
throw (InvalidSnapshotName) e;
if (e instanceof RuntimeException)
......@@ -596,4 +589,8 @@ class ArchiveImpl implements CDStarArchive {
throw new BackendError("Unable to create snapshot", e);
}
}
StorageObject getRawStorageObject() {
return sto;
}
}
......@@ -6,7 +6,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
......@@ -25,69 +25,76 @@ import de.gwdg.cdstar.runtime.client.exc.SnapshotLocked;
class AttributeCache {
private final ArchiveImpl a;
private final Map<String, Map<String, AttributeImpl>> attrMap;
private final ArchiveImpl archive;
private final SnapshotImpl snapshot; // null if not a snapshot
private Map<String, Map<String, AttributeImpl>> attrMapCache;
private Resource store;
private boolean modified = false;
private boolean modifiedInternal;
private boolean isSnapshot;
private Resource store;
AttributeJsonFormat<AttributeImpl> jsonFormat = new AttributeJsonFormat<AttributeImpl>() {
@Override
public AttributeImpl makeAttr(String fileId, String attrName, List<String> values) {
FileImpl file = null;
if (Utils.notNullOrEmpty(fileId))
file = a.getInternalFileById(fileId).orElseThrow(() -> new BackendError.DamagedDataError(
"Metadata table references unknown file id: " + fileId));
return new AttributeImpl(AttributeCache.this, file, attrName, values);
return new AttributeImpl(AttributeCache.this, attrName, fileId, values);
}
@Override
public List<String> getValues(AttributeImpl attr) {
if (attr.getFile() != null && attr.getFile().isRemoved())
return Collections.emptyList();
return attr.values();
}
};
public AttributeCache(ArchiveImpl a, Resource store, boolean isSnapshot) {
this.a = a;
this.store = store;
this.isSnapshot = isSnapshot;
public AttributeCache(ArchiveImpl archive, SnapshotImpl snapshot) {
this.archive = archive;
this.snapshot = snapshot;
}
private Map<String, Map<String, AttributeImpl>> load() {
if (attrMapCache != null)
return attrMapCache;
store = snapshot != null ? snapshot.getMetadataResource()
: archive.getResource(ArchiveImpl.METAFILE);
if (store == null || store.getSize() == 0) {
attrMap = new HashMap<>();
attrMapCache = new HashMap<>(2);
} else {
if (!ArchiveImpl.METAFILE_TYPE.equals(store.getMediaType()))
throw new BackendError("Failed to load metadata table. Wrong type");
try (InputStream is = new BufferedInputStream(Channels.newInputStream(store.getReadChannel(0)))) {
final JsonParser parser = SharedObjectMapper.json.getFactory().createParser(is);
attrMap = jsonFormat.parse(parser);
attrMapCache = jsonFormat.parse(parser);
parser.close();
} catch (final IOException e1) {
throw new BackendError("Failed to load metadata table", e1);
}
}
return attrMapCache;
}
synchronized void flush() {
if (!modified && !modifiedInternal)
if (snapshot != null || !modified && !modifiedInternal)
return; // Nothing to do
if (store != null && attrMap.isEmpty()) {
if (store != null && attrMapCache.isEmpty()) {
store.remove();
return;
}
if (store == null) {
store = a.createResource(ArchiveImpl.METAFILE);
store = archive.createResource(ArchiveImpl.METAFILE);
store.setMediaType(ArchiveImpl.METAFILE_TYPE);
}
try (BufferedOutputStream out = new BufferedOutputStream(Channels.newOutputStream(store.getWriteChannel(0)))) {
final JsonGenerator gen = SharedObjectMapper.json.getFactory().createGenerator(out);
jsonFormat.serialize(gen, attrMap);
jsonFormat.serialize(gen, attrMapCache);
gen.close();
} catch (final IOException e1) {
throw new BackendError("Failed to store metadata", e1);
} catch (final IOException e) {
throw new BackendError("Failed to store metadata", e);
}
}
......@@ -96,42 +103,57 @@ class AttributeCache {
throw new IllegalArgumentException("Not a valid attribute name: " + name);
final String key = file == null ? "" : file.getID();
return attrMap.computeIfAbsent(key, f -> new HashMap<>()).computeIfAbsent(name,
propName -> new AttributeImpl(this, file, name, new ArrayList<>()));
AttributeImpl attr = load().computeIfAbsent(key, f -> new HashMap<>()).computeIfAbsent(name,
propName -> new AttributeImpl(this, name, key, new ArrayList<>()));
return attr;
}
public synchronized Set<CDStarAttribute> getAttributes(FileImpl file) {
final String key = file == null ? "" : file.getID();
return new HashSet<>(attrMap.computeIfAbsent(key, f -> new HashMap<>()).values());
Collection<AttributeImpl> attrs = load().computeIfAbsent(key, f -> new HashMap<>()).values();
return new HashSet<>(attrs);
}
public synchronized Set<String> getAttributeNames(FileImpl file) {
final String key = file == null ? "" : file.getID();
return new HashSet<>(attrMap.computeIfAbsent(key, f -> new HashMap<>()).keySet());
return new HashSet<>(load().computeIfAbsent(key, f -> new HashMap<>()).keySet());
}
void requireModifiable() {
if (isSnapshot)
if (snapshot != null)
throw new SnapshotLocked();
a.requireTransactionWriteable();
a.requirePayloadWriteable();
a.checkPermission(ArchivePermission.CHANGE_META);
archive.requireTransactionWriteable();
archive.requirePayloadWriteable();
archive.checkPermission(ArchivePermission.CHANGE_META);
}
void setModifed(AttributeImpl attr) {
if (!modified) {
modified = true;
a.markContentModified();
archive.markContentModified();
}
if (Utils.notNullOrEmpty(attr.getFileRef())) {
String key = attr.getFileRef();
// TODO: This is a potentially expensive O(number-of-files) lookup,
// but most alternatives would make the common case worse. Better
// idea?
CDStarFile file = Utils.first(
snapshot == null ? archive.getInternalFileList() : snapshot.getInternalFileList(),
f -> key.equals(f.getID()));
archive.forEachListener(l -> l.filePropertyChanged(file, attr, attr.getOrigValues()));
} else {
archive.forEachListener(l -> l.propertyChanged(archive, attr, attr.getOrigValues()));
}
a.forEachListener(l -> l.propertyChanged(attr, attr.getFile(), attr.getOrigValues()));
}
/**
* Clears the attributes for a given file, if present. Does not check for
* permissions, as we assume that this is just a side-effect of removing a file.
* permissions, as we assume that this is just a side-effect of removing a
* file.
*/
void onFileRemoved(FileImpl file) {
if (attrMap.remove(file.getID()) != null) {
if (load().remove(file.getID()) != null) {
modifiedInternal = true;
}
}
......
......@@ -4,6 +4,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import de.gwdg.cdstar.Utils;
class AttributeImpl implements CDStarAttribute {
private static final String META_SEP = ":";
private static final String META_PREFIX = "meta" + META_SEP;
......@@ -12,21 +14,21 @@ class AttributeImpl implements CDStarAttribute {
private final List<String> origValues;
private final String name;
private final AttributeCache parent;
private final FileImpl file;
private final String fileRef;
public AttributeImpl(AttributeCache parent, FileImpl file, String name, List<String> values) {
public AttributeImpl(AttributeCache parent, String name, String fileRef, List<String> values) {
this.parent = parent;
this.file = file;
this.fileRef = Utils.notNullOrEmpty(fileRef) ? fileRef : null;
this.name = name;
origValues = values;
}
List<String> getOrigValues() {
return origValues;
String getFileRef() {
return fileRef;
}
CDStarFile getFile() {
return file;
List<String> getOrigValues() {
return origValues;
}
@Override
......@@ -55,8 +57,7 @@ class AttributeImpl implements CDStarAttribute {
* Split a fully qualified parameter name into namespace and local name. If
* no namespace is given, the "default" namespace is used.
*
* @param fullId
* Fully qualified field name.
* @param fullId Fully qualified field name.
* @return A string array with two elements: Namespace and fieldName.
*/
public static String[] splitFieldName(String fullId) {
......@@ -75,5 +76,4 @@ class AttributeImpl implements CDStarAttribute {
return name;
}
}
......@@ -24,17 +24,21 @@ class FileImpl implements CDStarFile {
private static final String DEFAULT_MEDIA_TYPE = "application/octet-stream";
static final String RESOURCE_PREFIX = "data/";
final ArchiveImpl archive;
final Resource resource;
private final Resource resource;
String name;
private final long oldSize;
private boolean isSnapshot;
private final SnapshotImpl snapshot;
FileImpl(ArchiveImpl archive, Resource store, String name, boolean isSnapshot) {
FileImpl(ArchiveImpl archive, Resource store, String name, SnapshotImpl snapshot) {
this.archive = archive;
this.resource = store;
resource = store;
this.name = name;
this.oldSize = store.getSize();
this.isSnapshot = isSnapshot;
oldSize = store.getSize();
this.snapshot = snapshot;
}
Resource getResource() {
return resource;
}
void flush() {
......@@ -53,6 +57,8 @@ class FileImpl implements CDStarFile {
@Override
public String getID() {
if (snapshot != null)
return resource.getProperty(SnapshotImpl.PROP_FILE_ID);
return resource.getId();
}
......@@ -118,7 +124,7 @@ class FileImpl implements CDStarFile {
@Override
public void setName(String name) throws FileExists, InvalidFileName {
if (isSnapshot)
if (snapshot != null)
throw new SnapshotLocked();
archive.requireTransactionWriteable();
......@@ -157,7 +163,7 @@ class FileImpl implements CDStarFile {
@Override
public void setMediaType(String type, String coding) {
if (isSnapshot)
if (snapshot != null)
throw new SnapshotLocked();
archive.requireTransactionWriteable();
archive.requirePayloadWriteable();
......@@ -218,7 +224,7 @@ class FileImpl implements CDStarFile {
@Override
public CDStarFile truncate(long length) throws StaleHandle, ExternalResourceException, IOException {
if (isSnapshot)
if (snapshot != null)
throw new SnapshotLocked();
archive.requireTransactionWriteable();
archive.requirePayloadWriteable();
......@@ -245,7 +251,7 @@ class FileImpl implements CDStarFile {
@Override
public WritableByteChannel getWriteChannel() throws StaleHandle, ExternalResourceException, IOException {
if (isSnapshot)
if (snapshot != null)
throw new SnapshotLocked();
archive.requireTransactionWriteable();
archive.checkPermission(ArchivePermission.CHANGE_FILES);
......@@ -279,7 +285,7 @@ class FileImpl implements CDStarFile {
@Override
public void remove() {
if (isSnapshot)
if (snapshot != null)
throw new SnapshotLocked();
archive.removeFile(this);
archive.forEachListener(l -> l.fileRemoved(this));
......@@ -291,12 +297,18 @@ class FileImpl implements CDStarFile {
@Override
public CDStarAttribute getAttribute(String name) {
return archive.getMeta().getAttribute(this, name.toString());
if (snapshot != null)
return snapshot.getAttributeCache().getAttribute(this, name.toString());
else
return archive.getMeta().getAttribute(this, name.toString());
}
@Override
public Set<CDStarAttribute> getAttributes() {
return archive.getMeta().getAttributes(this);
if (snapshot != null)
return snapshot.getAttributeCache().getAttributes(this);
else
return archive.getMeta().getAttributes(this);
}
void setExternalLocation(String hint) {
......
......@@ -9,25 +9,30 @@ import de.gwdg.cdstar.pool.StorageObject;
/**
* Metadata about a single snapshot. This info is stored in the
* {@link SnapshotList#SNAPSHOTFILE} and partly in properties of the snapshot
* {@link StorageObject} itself.
* {@link SnapshotList#SNAPSHOT_INDEX_FILE}.and partly in properties of the
* snapshot {@link StorageObject} itself.
*/
public class SnapshotEntry {
public final String id; // The StorageObject ID of the snapshot, NOT the id of the source
public final String name;
public final String archive; // The archive ID this snapshot was taken from.
public final String revision;
public final String name; // Snapshot name
public String ref; // ID or location of the snapshot data object, or null
// for local snapshots.
public final Date created;
public final String creator;
public Date removed; // Time this snapshot was removed, or null
@JsonCreator
public SnapshotEntry(@JsonProperty("id") String id, @JsonProperty("revision") String revision,
@JsonProperty("name") String name, @JsonProperty("created") Date created,
@JsonProperty("creator") String creator) {
this.id = id;
public SnapshotEntry(@JsonProperty("archive") String archive, @JsonProperty("revision") String revision,