Commit bf6c02ce authored by mhellka's avatar mhellka
Browse files

Cleaning up SnapshotImpl and others.

parent e6b4a00d
Pipeline #134411 passed with stage
in 5 minutes and 10 seconds
......@@ -71,13 +71,6 @@ public interface CDStarArchive extends CDStarAnnotateable {
return getStorageState().setProfile(profile);
};
/**
* Return true if the archive is locked (write protected). The locking only
* affects the payload of the archive (files and meta-attributes), not the
* administrative attributes (owner, ACLs, ...).
*/
boolean isLocked();
/**
* Return the archive storage state.
*/
......
......@@ -212,16 +212,16 @@ class ArchiveImpl implements CDStarArchive {
void requirePayloadAvailable() {
requireTransactionAlive();
if (!isReadable())
if (!getStorageState().isReadable())
throw new ArchiveNotAvailable(vault.getName(), getId());
}
void requirePayloadWriteable() {
requireTransactionWriteable();
if (isLocked())
if (!getStorageState().isWriteable())
throw new ArchiveLocked(vault.getName(), getId());
if (!isReadable())
if (!getStorageState().isReadable())
throw new ArchiveNotAvailable(vault.getName(), getId());
}
......@@ -250,15 +250,6 @@ class ArchiveImpl implements CDStarArchive {
return String.valueOf(isContentModified ? xRevision + 1 : xRevision);
}
@Override
public boolean isLocked() {
return !getStorageState().isWriteable();
}
public boolean isReadable() {
return getStorageState().isReadable();
}
@Override
public synchronized CDStarStorageState getStorageState() {
if (xStorageStateCache == null)
......@@ -468,8 +459,6 @@ class ArchiveImpl implements CDStarArchive {
@Override
public void setProperty(String name, String value) {
requireTransactionWriteable();
markModified();
setRawProperty(PROP_SYS_NS + name, value);
}
......@@ -478,6 +467,7 @@ class ArchiveImpl implements CDStarArchive {
}
void setRawProperty(String name, String value) {
requireTransactionWriteable();
sto.setProperty(name, value);
markModified();
}
......@@ -492,9 +482,8 @@ class ArchiveImpl implements CDStarArchive {
}
synchronized SnapshotList getSnapshotsInternal() {
if (snapshotList != null)
return snapshotList;
snapshotList = new SnapshotList(this);
if (snapshotList == null)
snapshotList = new SnapshotList(this);
return snapshotList;
}
......@@ -516,6 +505,9 @@ class ArchiveImpl implements CDStarArchive {
requireTransactionWriteable();
requirePayloadAvailable();
if (isContentModified())
throw new IllegalStateException("Snapshots can only be created from unmodified content");
try {
final CDStarSnapshot snap = getSnapshotsInternal().createSnapshot(name);
markModified();
......
......@@ -312,14 +312,6 @@ class FileImpl implements CDStarFile {
return archive.getMeta().getAttributes(this);
}
String getExternalLocation() {
return resource.getExternalLocation();
}
void restoreExternal(InputStream data) throws IOException {
resource.restoreExternal(data);
}
void freeze() {
CDStarStorageState state = getSnapshot().map(s -> s.getStorageState())
.orElseGet(getArchive()::getStorageState);
......
package de.gwdg.cdstar.runtime.client;
import java.util.Date;
import java.time.Instant;
import java.util.HashMap;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.gwdg.cdstar.pool.StorageObject;
/**
* Metadata about a single snapshot. This info is stored in the
* {@link SnapshotList#SNAPSHOT_INDEX_FILE}.and partly in properties of the
* snapshot {@link StorageObject} itself.
* Metadata about a single snapshot. Stored in the
* {@link SnapshotList#SNAPSHOT_INDEX_FILE}.
*/
public class SnapshotEntry {
public final String archive; // The archive ID this snapshot was taken from.
public final String revision;
public final String name; // Snapshot name
public String ref; // ID or location of the snapshot data object, or null
// for local snapshots.
public final Date created;
public final String revision; // Revision of the archive at the time the snapshot was taken
public final String name; // Snapshot name (unique per archive)
public final String ref; // ID of the snapshot data object.
public final Instant created;
public final String creator;
public Date modified; // Time this snapshot was last modified (e.g. profile)
public Date removed; // Time this snapshot was removed, or null
public Map<String, String> properties;
public Instant modified; // Time this snapshot was last modified (e.g. profile)
public Instant removed; // Time this snapshot was removed, or null
public Map<String, String> properties; // (internal) properties. For example, profile or migration state.
@JsonCreator
public SnapshotEntry(@JsonProperty("archive") String archive, @JsonProperty("revision") String revision,
@JsonProperty("name") String name, @JsonProperty("creator") String creator,
@JsonProperty("created") Date created, @JsonProperty("modified") Date modified,
@JsonProperty("removed") Date removed, @JsonProperty("ref") String ref,
@JsonProperty("created") Instant created, @JsonProperty("modified") Instant modified,
@JsonProperty("removed") Instant removed, @JsonProperty("ref") String ref,
@JsonProperty("properties") Map<String, String> properties) {
this.archive = archive;
this.revision = revision;
......
package de.gwdg.cdstar.runtime.client;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
......@@ -10,11 +9,9 @@ import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
import de.gwdg.cdstar.SharedObjectMapper;
import de.gwdg.cdstar.Utils;
import de.gwdg.cdstar.pool.BackendError;
import de.gwdg.cdstar.pool.PoolError.ExternalResourceException;
import de.gwdg.cdstar.pool.PoolError.NameConflict;
import de.gwdg.cdstar.pool.PoolError.StaleHandle;
import de.gwdg.cdstar.pool.Resource;
import de.gwdg.cdstar.pool.StorageObject;
......@@ -23,33 +20,12 @@ import de.gwdg.cdstar.runtime.client.exc.ArchiveNotAvailable;
import de.gwdg.cdstar.runtime.client.exc.FileNotFound;
import de.gwdg.cdstar.runtime.client.exc.InvalidSnapshotName;
/**
* Snapshots are either stored locally in the same storage object as the source
* archive, or in separate storage objects referenced by
* {@link SnapshotEntry#ref}. {@link Resource}s are stored under a
* {@code /snapshots/{snapshotName}/} prefix and {@link StorageObject}
* properties are prefixed with {@code snapshot:{snapshotName}:} to allow
* multiple snapshots to be stored in the same storage object. For this reason,
* snapshot names should be short and must not contain slashes or colons.
*
* All {@link SnapshotEntry}s are listed in {@code /snapshots.json} in the
* storage object of the source archive. There is also a
* {@code /snapshots/{snapshotName}/snapshot.json} next to each snapshot with
* the same information.
*
* Snapshots are immutable, with the exception that they can be removed.
* Removing a snapshot frees any resources, but keeps the entry in the
* {@code snapshots.json} list so the same name cannot be referenced again.
*/
class SnapshotImpl implements CDStarSnapshot {
private static final String PATH_PREFIX = "snapshots/";
private static final String PATH_SEP = "/";
private static final String PROP_PREFIX = ArchiveImpl.PROP_NS + "snapshot:";
private static final String PROP_SEP = ":";
static final String PROP_FILE_ID = PROP_PREFIX + "fid";
static final String MIME_TYPE = "application/x-cdstar-snapshot;v=3";
static final String PROP_FILE_ID = "cdstar:fid";
static final Pattern NAME_PATTERN = Pattern.compile("^[a-z0-9-.]+$");
private final ArchiveImpl head;
......@@ -72,68 +48,44 @@ class SnapshotImpl implements CDStarSnapshot {
this.snap = meta;
}
private static String getPropName(String snapshotName, String propName) {
return PROP_PREFIX + snapshotName + PROP_SEP + propName;
}
private static String getResourceName(String snapshotName, String resourceName) {
return PATH_PREFIX + snapshotName + PATH_SEP + resourceName;
}
public boolean isLocal() {
return Utils.nullOrEmpty(snap.ref);
return Utils.equal(head.getId(), snap.ref);
}
public static boolean isValidName(String name) {
static boolean isValidName(String name) {
// MUST NOT requirements
if (name.contains(PROP_SEP) || name.contains(PATH_SEP))
if (name.contains(PATH_SEP))
return false;
// These can be loosened later
return name.matches("^[a-z0-9-.]+$");
return NAME_PATTERN.matcher(name).matches();
}
/**
* Create and return a new snapshot. Called by
* {@link SnapshotList#createSnapshot(String)} in a synchronized way.
*/
static SnapshotImpl prepare(ArchiveImpl head, String name, StorageObject target)
static SnapshotImpl create(ArchiveImpl head, SnapshotEntry meta, StorageObject target)
throws InvalidSnapshotName, StaleHandle, ExternalResourceException, IOException {
head.requireTransactionWriteable();
head.requirePayloadAvailable();
if (head.isContentModified())
throw new IllegalStateException("Snapshots can only be created from unmodified content");
if (!SnapshotImpl.isValidName(name))
throw new InvalidSnapshotName(name, "Unsupported characters");
boolean local = head.getRawStorageObject().getId().equals(target.getId());
boolean isSeparate = !meta.ref.equals(meta.archive);
try {
String ref = local ? null : target.getId();
String creator = head.getSession().getClient().getSubject().getPrincipal().getFullId();
Date now = new Date();
SnapshotEntry meta = new SnapshotEntry(head.getId(), head.getRev(), name, creator, now, now, null, ref,
null);
// Store snapshot metadata in non-local snapshots
Resource snapMetaResource = target.createResource(getResourceName(name, "snapshot.json"));
try (WritableByteChannel wc = snapMetaResource.getWriteChannel()) {
wc.write(ByteBuffer.wrap(SharedObjectMapper.json.writeValueAsBytes(meta)));
}
// Copy metadata (bit for bit, to allow deduplication)
Resource metaOrig = head.getResource(ArchiveImpl.METAFILE);
if (metaOrig != null) {
Resource metaResource = target.createResource(getResourceName(name, ArchiveImpl.METAFILE));
Resource metaResource = target.createResource(getResourceName(meta.name, ArchiveImpl.METAFILE));
metaResource.setMediaType(ArchiveImpl.METAFILE_TYPE);
metaResource.copyFrom(metaOrig);
}
// Copy files
for (FileImpl r : head.getInternalFileList()) {
Resource t = target.createResource(getResourceName(name, FileImpl.RESOURCE_PREFIX + r.getName()));
Resource t = target.createResource(getResourceName(meta.name, FileImpl.RESOURCE_PREFIX + r.getName()));
t.setMediaType(r.getMediaType());
t.setProperty(PROP_FILE_ID, r.getID());
t.copyFrom(r.getResource());
......@@ -141,12 +93,11 @@ class SnapshotImpl implements CDStarSnapshot {
SnapshotImpl snap = new SnapshotImpl(head, meta);
snap.isNew = true;
return snap;
} catch (StaleHandle | ExternalResourceException | NameConflict | IOException e) {
} catch (Exception e) {
// Cleanup anything we just copied, and completely remove the STO if it is
// otherwise empty.
purgeSnapshot(head, target, name, !local);
purgeSnapshot(target, meta.name, isSeparate);
throw e;
}
}
......@@ -162,20 +113,14 @@ class SnapshotImpl implements CDStarSnapshot {
* @param removeIfEmpty If true, remove the {@link StorageObject} if no files or
* properties are left.
*/
private static void purgeSnapshot(ArchiveImpl head, StorageObject target, String snapshotName,
private static void purgeSnapshot(StorageObject target, String snapshotName,
boolean removeIfEmpty) {
String resourcePrefix = getResourceName(snapshotName, "");
target.getResourcesByPrefix(resourcePrefix).forEach(Resource::remove);
// Delete snapshot properties from head.
String propPrefix = getPropName(snapshotName, "");
head.getRawPropertyNames().stream()
.filter(n -> n.startsWith(propPrefix))
.forEach(name -> head.setRawProperty(name, null));
if (removeIfEmpty
&& target.getResources().stream().noneMatch(r -> !r.isRemoved())
&& target.getPropertyNames().isEmpty()) {
&& target.getType().equals(SnapshotList.SNAPSHOT_TYPE)
&& target.getResources().stream().noneMatch(r -> !r.isRemoved())) {
target.remove();
}
......@@ -185,26 +130,20 @@ class SnapshotImpl implements CDStarSnapshot {
if (cachedSto != null)
return cachedSto;
if (isLocal()) {
cachedSto = head.getRawStorageObject();
} else {
cachedSto = ((VaultImpl) head.getVault()).loadSTO(snap.ref);
if (cachedSto == null)
throw new BackendError("Snapshot not found");
if (!Utils.equal(cachedSto.getType(), MIME_TYPE))
throw new BackendError("Snapshot mime-type mismatch: " + cachedSto.getType());
}
cachedSto = ((VaultImpl) head.getVault()).loadSTO(snap.ref);
if (cachedSto == null)
throw new BackendError("Snapshot storage object not found: " + snap.ref);
return cachedSto;
}
String getProperty(String name) {
String getRawProperty(String name) {
return snap.properties.get(name);
}
void setProperty(String name, String value) {
void setRawProperty(String name, String value) {
modified = true;
snap.modified = new Date();
snap.modified = Instant.now();
if (value == null)
snap.properties.remove(name);
else
......@@ -235,12 +174,12 @@ class SnapshotImpl implements CDStarSnapshot {
@Override
public Date getCreated() {
return snap.created;
return Date.from(snap.created);
}
@Override
public Date getModified() {
return snap.modified;
return Date.from(snap.modified);
}
@Override
......@@ -318,20 +257,16 @@ class SnapshotImpl implements CDStarSnapshot {
if (getStorageState().isMirrored())
throw new ArchiveNotAvailable(head);
// TODO: Snapshots that are created and removed in the same transaction
// should be allowed to just disappear, not block the snapshot name
// forever.
head.checkPermission(ArchivePermission.DELETE);
head.markModified();
// Remove all files and properties belonging to this snapshot. Remove non-local
// snapshot STOs if empty and thus no longer needed.
purgeSnapshot(head, load(), getName(), !isLocal());
purgeSnapshot(load(), getName(), !isLocal());
cachedFiles = null;
cachedMetadata = null;
snap.removed = new Date();
snap.removed = Instant.now();
modified = true;
head.forEachListener(l -> l.snapshotRemoved(this));
}
......
......@@ -5,6 +5,7 @@ import java.io.InputStream;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
......@@ -24,6 +25,30 @@ import de.gwdg.cdstar.pool.Resource;
import de.gwdg.cdstar.pool.StorageObject;
import de.gwdg.cdstar.runtime.client.exc.InvalidSnapshotName;
/**
* {@link SnapshotList} is responsible for creating and managing
* {@link CDStarSnapshot}s.
*
* Administrative metadata about snapshots is stored as a list of
* {@link SnapshotEntry}s in {@value #SNAPSHOT_INDEX_FILE}. The actual payload
* (data files and {@value ArchiveImpl#METAFILE}) is stored either locally in
* the same {@link StorageObject}, or in separate {@link StorageObject}
* referenced via {@link SnapshotEntry#ref}.
*
* Separate snapshot {@link StorageObject}s are created with the
* {@value SnapshotList#SNAPSHOT_TYPE} type and have a
* {@link SnapshotList#SNAPSHOT_BACKREF_PROP} property that back-references to
* the source archive. Snapshots from different archives are never mixed into
* the same {@link StorageObject}.
*
* In both cases, local or separate, all {@link Resource} names are prefixed
* with {@code /snapshots/{snapshotName}/} to allow multiple snapshots in a
* single {@link StorageObject}. For this reason, snapshot names must not
* contain the past separator.
*
* Snapshot payload is immutable, with the exception that removing a snapshot
* also removes the data files from the {@link StorageObject}.
*/
class SnapshotList {
private static final Logger log = LoggerFactory.getLogger(SnapshotList.class);
......@@ -31,9 +56,11 @@ class SnapshotList {
public static final String SNAPSHOT_INDEX_FILE = "snapshots.json";
public static final String SNAPSHOT_INDEX_TYPE = "application/x-cdstar-snapshotlist;v=3";
public static final String SNAPSHOT_TYPE = "application/x-cdstar-snapshot;v=3";
public static final String SNAPSHOT_BACKREF_PROP = "cdstar:snapshot:source";
List<SnapshotImpl> loaded;
private final ArchiveImpl archive;
private Resource snapshotIndex;
private int resourceCountThreshold = 1024;
SnapshotList(ArchiveImpl source) {
......@@ -49,19 +76,23 @@ class SnapshotList {
*/
synchronized CDStarSnapshot createSnapshot(String name)
throws InvalidSnapshotName, StaleHandle, ExternalResourceException, IOException {
archive.requireTransactionWriteable();
archive.requirePayloadAvailable();
if (archive.isContentModified())
throw new IllegalStateException("Snapshots can only be created from unmodified content");
if (getSnapshot(name).isPresent())
throw new InvalidSnapshotName(name, "Snapshot with that name already exists");
if (!SnapshotImpl.isValidName(name))
throw new InvalidSnapshotName(name, "Invalid snapshot name");
StorageObject target = findOrCreateSnapshotTarget();
log.debug("Storing snapshot {} of {} in {}", archive, target.getId());
log.debug("Storing snapshot {}@{} in {}", archive, name, target.getId());
Instant now = Instant.now();
SnapshotEntry meta = new SnapshotEntry(
archive.getId(), archive.getRev(), name,
archive.getSession().getClient().getSubject().getPrincipal().getFullId(),
now, now, null, target.getId(), null);
SnapshotImpl snap = SnapshotImpl.prepare(archive, name, target);
SnapshotImpl snap = SnapshotImpl.create(archive, meta, target);
loaded.add(snap);
archive.forEachListener(l -> l.snapshotCreated(snap));
......@@ -86,7 +117,8 @@ class SnapshotList {
.findFirst()
.orElseGet(() -> {
StorageObject sto = ((VaultImpl) archive.getVault()).createSTO();
sto.setType(SnapshotImpl.MIME_TYPE);
sto.setType(SNAPSHOT_TYPE);
sto.setProperty(SNAPSHOT_BACKREF_PROP, archive.getId());
return sto;
});
}
......@@ -106,6 +138,7 @@ class SnapshotList {
return;
// Create snapshot index resource if needed
Resource snapshotIndex = archive.getResource(SNAPSHOT_INDEX_FILE);
if (snapshotIndex == null) {
snapshotIndex = archive.createResource(SNAPSHOT_INDEX_FILE);
snapshotIndex.setMediaType(SNAPSHOT_INDEX_TYPE);
......@@ -129,7 +162,7 @@ class SnapshotList {
return loaded;
loaded = new ArrayList<>();
snapshotIndex = archive.getResource(SNAPSHOT_INDEX_FILE);
Resource snapshotIndex = archive.getResource(SNAPSHOT_INDEX_FILE);
if (snapshotIndex == null)
return loaded;
......
......@@ -13,7 +13,7 @@ import de.gwdg.cdstar.runtime.lts.LTSConfig;
class StorageStateImpl implements CDStarStorageState {
private final ArchiveImpl archive;
private SnapshotImpl snapshot;
private final SnapshotImpl snapshot;
public StorageStateImpl(ArchiveImpl archive) {
this(archive, null);
......@@ -26,14 +26,14 @@ class StorageStateImpl implements CDStarStorageState {
private Optional<String> getRawProperty(String name) {
if (snapshot != null)
return Optional.ofNullable(snapshot.getProperty(name));
return Optional.ofNullable(snapshot.getRawProperty(name));
return Optional.ofNullable(archive.getRawProperty(name));
}
private void setRawProperty(String name, String value) {
archive.requireTransactionWriteable();
if (snapshot != null)
snapshot.setProperty(name, value);
snapshot.setRawProperty(name, value);
else
archive.setRawProperty(name, value);
}
......@@ -61,7 +61,7 @@ class StorageStateImpl implements CDStarStorageState {
return isAllFilesAvailable();
}
boolean isAllFilesAvailable() {
private boolean isAllFilesAvailable() {
for (FileImpl file : (snapshot == null ? archive.getInternalFileList() : snapshot.getInternalFileList()))
if (file.isFrozen())
return false;
......
......@@ -53,7 +53,7 @@ public class SnapshotTest extends RuntimeBaseTest {
}
boolean isLocal(CDStarSnapshot snap) {
return ((SnapshotImpl) snap).snap.ref == null;
return ((SnapshotImpl) snap).isLocal();
}
@Test
......@@ -207,8 +207,8 @@ public class SnapshotTest extends RuntimeBaseTest {
commit();
VaultImpl v = (VaultImpl) getVault();
assertEquals(SnapshotImpl.MIME_TYPE, v.loadSTO(stoRef1).getType());
assertEquals(SnapshotImpl.MIME_TYPE, v.loadSTO(stoRef2).getType());
assertEquals(SnapshotList.SNAPSHOT_TYPE, v.loadSTO(stoRef1).getType());
assertEquals(SnapshotList.SNAPSHOT_TYPE, v.loadSTO(stoRef2).getType());
rollback();
// Removing a snapshot also removes its storage object
......@@ -218,7 +218,7 @@ public class SnapshotTest extends RuntimeBaseTest {
VaultImpl v2 = (VaultImpl) getVault();
TestUtils.assertRaises(PoolError.NotFound.class, () -> v2.loadSTO(stoRef1));
assertEquals(SnapshotImpl.MIME_TYPE, v.loadSTO(stoRef2).getType());
assertEquals(SnapshotList.SNAPSHOT_TYPE, v.loadSTO(stoRef2).getType());
v2.loadSTO(stoRef2);
// Removing an archive also removes all snapshots
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment