Commit fb4b8495 authored by mhellka's avatar mhellka
Browse files

Implemented REST apis to create/list/access snapshots (incomplete).

parent 462f42cc
......@@ -17,6 +17,9 @@ import de.gwdg.cdstar.rest.utils.form.FormPart;
import de.gwdg.cdstar.rest.utils.form.MultipartParser;
import de.gwdg.cdstar.rest.utils.form.UrlEncodedParser;
/**
* Synchronous form parser for small forms with no file uploads.
*/
public class FormHelper {
private final Map<String, String> parts;
......@@ -29,6 +32,10 @@ public class FormHelper {
return Optional.ofNullable(parts.get(name));
}
public String get(String name, String defaultValue) {
return get(name).orElse(defaultValue);
}
/**
* Parse the entire request in a blocking fashion.
*
......@@ -44,14 +51,14 @@ public class FormHelper {
* @throws IOException
*/
public static FormHelper parse(RestContext ctx, int maxElementSize, int maxRead)
throws EntityTooLarge, UnsupportedContentType, FormFieldTooLarge, FormParserException, IOException {
throws FormParserException {
if (!ctx.hasEntity())
return new FormHelper(Collections.emptyMap());
final long clen = ctx.getContentLength();
if (clen > 0) {
if (clen > maxRead)
throw new EntityTooLarge();
throw new EntityTooLarge(maxRead);
maxRead = (int) Math.min(maxRead, clen);
}
......@@ -61,7 +68,7 @@ public class FormHelper {
} else if (ctx.isForm()) {
form = new UrlEncodedParser(maxElementSize, true, StandardCharsets.UTF_8);
} else {
throw new UnsupportedContentType();
throw new UnsupportedContentType(ctx.getContentType());
}
final Map<String, String> result = new HashMap<>();
......@@ -70,9 +77,14 @@ public class FormHelper {
int total = 0;
boolean eof = false;
do {
final int chunk = ctx.read(buffer);
int chunk;
try {
chunk = ctx.read(buffer);
} catch (final IOException e) {
throw new FormParserException("Unexpected EOF or read error", e);
}
if ((total += chunk) > maxRead)
throw new EntityTooLarge();
throw new EntityTooLarge(maxRead);
buffer.flip();
final List<FormPart> parts;
......@@ -85,26 +97,38 @@ public class FormHelper {
}
for (final FormPart part : parts) {
if (part.isComplete())
result.put(part.getName(), part.drainToString(StandardCharsets.UTF_8));
else if (part.getBuffered() >= maxElementSize)
throw new FormFieldTooLarge(part.getName());
if (!part.isComplete()) {
if (part.getBuffered() >= maxElementSize)
throw new FormFieldTooLarge(part.getName());
continue;
}
if (part.getName() == null)
throw new FormParserException("Unnamed field");
result.put(part.getName(), part.drainToString(StandardCharsets.UTF_8));
}
} while (!eof);
return new FormHelper(result);
}
public static class EntityTooLarge extends IOException {
public static class EntityTooLarge extends FormParserException {
private static final long serialVersionUID = 4874186929008883638L;
public EntityTooLarge(int limit) {
super("Larger than " + limit);
}
}
public static class UnsupportedContentType extends IOException {
public static class UnsupportedContentType extends FormParserException {
private static final long serialVersionUID = -432047215473060595L;
public UnsupportedContentType(String type) {
super(type);
}
}
public static class FormFieldTooLarge extends IOException {
public static class FormFieldTooLarge extends FormParserException {
private static final long serialVersionUID = 6187007509004252521L;
private final String fieldName;
public FormFieldTooLarge(String name) {
......
......@@ -11,6 +11,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
......@@ -69,9 +70,11 @@ import de.gwdg.cdstar.web.common.model.FileInfo;
import de.gwdg.cdstar.web.common.model.FileList;
import de.gwdg.cdstar.web.common.model.MetaMap;
import de.gwdg.cdstar.web.common.model.SnapshotInfo;
import de.gwdg.cdstar.web.common.model.SnapshotList;
public class ArchiveEndpoint implements RestBlueprint {
private static final String SNAPSHOT_SEP = "@";
private static final Set<String> withOptions = new HashSet<>(Arrays.asList("files", "meta", "acl"));
private RequestThrottle zipBreaker;
......@@ -112,7 +115,7 @@ public class ArchiveEndpoint implements RestBlueprint {
* stripped away before loading the archive.
*/
static CDStarArchive openArchive(CDStarVault vault, String archiveRef) throws ArchiveNotFound {
int i = archiveRef.indexOf("@");
final int i = archiveRef.indexOf(SNAPSHOT_SEP);
if (i > 0)
archiveRef = archiveRef.substring(0, i);
return vault.loadArchive(archiveRef);
......@@ -124,34 +127,34 @@ public class ArchiveEndpoint implements RestBlueprint {
*
* @throws SnapshotNotFound if a was referenced, but could not be resolved.
*/
static CDStarSnapshot resolveSnapshot(CDStarArchive archive, String archiveRef) throws SnapshotNotFound {
int i = archiveRef.indexOf("@");
static Optional<CDStarSnapshot> resolveSnapshot(CDStarArchive archive, String archiveRef) throws SnapshotNotFound {
final int i = archiveRef.indexOf(SNAPSHOT_SEP);
if (i > 0) {
String snapName = archiveRef.substring(i + 1);
return archive.getSnapshot(snapName).orElseThrow(() -> new SnapshotNotFound(snapName));
final String snapName = archiveRef.substring(i + 1);
return Optional.of(archive.getSnapshot(snapName).orElseThrow(() -> new SnapshotNotFound(snapName)));
}
return null;
return Optional.empty();
}
static void throwIfHasSnapshot(String archiveRef) {
if (archiveRef.contains("@"))
if (archiveRef.contains(SNAPSHOT_SEP))
throw new SnapshotLocked();
}
public Object handleGet(RestContext ctx)
throws NotAcceptable, IOException, ArchiveNotFound, VaultNotFound, SnapshotNotFound {
throws NotAcceptable, IOException, ArchiveNotFound, VaultNotFound, SnapshotNotFound {
final QueryHelper qh = new QueryHelper(ctx);
final String subResource = qh.getExclusiveParam("acl", "meta", "files", "export");
final String subResource = qh.getExclusiveParam("acl", "meta", "files", "export", "snapshots");
if ("export".equals(subResource)) {
zipBreaker.handleThrottled(ctx.startAsync(), 1, this::handleZipExport);
return null;
}
CDStarVault vault = SessionHelper.getCDStarSession(ctx, true).getVault(ctx.getPathParam("vault"));
CDStarArchive archive = openArchive(vault, ctx.getPathParam("archive"));
CDStarSnapshot snapshot = resolveSnapshot(archive, ctx.getPathParam("archive"));
final CDStarVault vault = SessionHelper.getCDStarSession(ctx, true).getVault(ctx.getPathParam("vault"));
final CDStarArchive archive = openArchive(vault, ctx.getPathParam("archive"));
final Optional<CDStarSnapshot> snapshot = resolveSnapshot(archive, ctx.getPathParam("archive"));
if ("acl".equals(subResource)) {
final boolean groupBySet = !qh.getOption("acl", "", "group", "explode").equals("explode");
......@@ -161,17 +164,30 @@ public class ArchiveEndpoint implements RestBlueprint {
if ("meta".equals(subResource)) {
qh.ensureNoUnusedParameters();
return getArchiveMeta(snapshot == null ? archive.getAttributes() : snapshot.getAttributes());
Set<CDStarAttribute> attrs = snapshot
.map(s -> s.getAttributes())
.orElseGet(() -> archive.getAttributes());
return getArchiveMeta(attrs);
}
if ("files".equals(subResource)) {
List<CDStarFile> files = snapshot == null ? archive.getFiles() : snapshot.getFiles();
int total = files.size();
Stream<FileInfo> filtered = getFilesFiltered(files, qh);
final List<CDStarFile> files = snapshot
.map(CDStarSnapshot::getFiles)
.orElseGet(() -> archive.getFiles());
final int total = files.size();
final Stream<FileInfo> filtered = getFilesFiltered(files, qh);
qh.ensureNoUnusedParameters();
return new FileList(filtered.collect(Collectors.toList()), total);
}
if ("snapshots".equals(subResource)) {
final List<SnapshotInfo> snapshots = archive.getSnapshots().stream()
.map(this::snapshotInfo)
.collect(Collectors.toList());
qh.ensureNoUnusedParameters();
return new SnapshotList(snapshots, snapshots.size());
}
// No subresource request
final Set<String> with = qh.getCsvOptions("with", withOptions);
......@@ -184,19 +200,19 @@ public class ArchiveEndpoint implements RestBlueprint {
final ArchiveInfo info = new ArchiveInfo();
info.id = archive.getId();
info.vault = vault.getName();
info.revision = snapshot == null ? archive.getRev() : snapshot.getRevision();
info.revision = snapshot.map(s -> s.getRevision()).orElse(archive.getRev());
info.created = archive.getCreated();
info.modified = snapshot == null ? archive.getContentModified() : snapshot.getCreated();
info.modified = snapshot.map(s -> s.getCreated()).orElse(archive.getContentModified());
info.owner = archive.getOwner();
info.file_count = snapshot == null ? archive.getFileCount() : snapshot.getFileCount();
info.file_count = snapshot.map(s -> s.getFileCount()).orElse(archive.getFileCount());
// TODO: Snapshots should have their own profile and state
info.profile = archive.getProfile().getName();
info.state = ModelHelper.getArchiveState(archive);
if (snapshot != null && info.state == ArchiveState.OPEN)
info.state = ArchiveState.LOCKED;
if (info.state == ArchiveState.OPEN && snapshot.isPresent())
info.state = ArchiveState.LOCKED; // Snapshots are always locked
ctx.header("Last-Modified", archive.getModified());
ctx.header("Last-Modified", archive.getModified()); // NOT getContentModified
if (showACL) {
try {
......@@ -208,8 +224,9 @@ public class ArchiveEndpoint implements RestBlueprint {
if (showMeta) {
try {
Set<CDStarAttribute> meta = snapshot != null ? snapshot.getAttributes() : archive.getAttributes();
info.meta = getArchiveMeta(meta);
info.meta = getArchiveMeta(snapshot
.map(CDStarSnapshot::getAttributes)
.orElseGet(() -> archive.getAttributes()));
} catch (final AccessError e) {
// Just keep it blank
}
......@@ -217,7 +234,9 @@ public class ArchiveEndpoint implements RestBlueprint {
if (showFiles) {
try {
List<CDStarFile> files = snapshot != null ? snapshot.getFiles() : archive.getFiles();
final List<CDStarFile> files = snapshot
.map(CDStarSnapshot::getFiles)
.orElseGet(() -> archive.getFiles());
info.files = getFilesFiltered(files, qh).collect(Collectors.toList());
} catch (final AccessError e) {
// Just keep it blank
......@@ -227,6 +246,14 @@ public class ArchiveEndpoint implements RestBlueprint {
return info;
}
SnapshotInfo snapshotInfo(final CDStarSnapshot snap) {
final SnapshotInfo info = new SnapshotInfo();
info.name = snap.getName();
info.creator = snap.getCreator();
info.created = snap.getCreated();
return info;
}
private Void handleZipExport(AsyncContext ac) throws ArchiveNotFound, VaultNotFound, SnapshotNotFound {
final QueryHelper qh = new QueryHelper(ac.getRequest());
......@@ -247,11 +274,12 @@ public class ArchiveEndpoint implements RestBlueprint {
throw Utils.wtf();
}
RestContext ctx = ac.getRequest();
CDStarVault vault = SessionHelper.getCDStarSession(ctx, true).getVault(ctx.getPathParam("vault"));
CDStarArchive archive = openArchive(vault, ctx.getPathParam("archive"));
CDStarSnapshot snapshot = resolveSnapshot(archive, ctx.getPathParam("archive"));
final List<CDStarFile> files = new ArrayList<>(snapshot == null ? archive.getFiles() : snapshot.getFiles());
final RestContext ctx = ac.getRequest();
final CDStarVault vault = SessionHelper.getCDStarSession(ctx, true).getVault(ctx.getPathParam("vault"));
final CDStarArchive archive = openArchive(vault, ctx.getPathParam("archive"));
final Optional<CDStarSnapshot> snapshot = resolveSnapshot(archive, ctx.getPathParam("archive"));
final List<CDStarFile> files = new ArrayList<>(
snapshot.map(CDStarSnapshot::getFiles).orElseGet(() -> archive.getFiles()));
// Handle include and exclude rules
final List<GlobPattern> include = qh.getFiltered("include", (s) -> new GlobPattern(s));
......@@ -259,7 +287,8 @@ public class ArchiveEndpoint implements RestBlueprint {
final IncludeExcludeFilter<String> filter = new IncludeExcludeFilter<>(include, exclude);
files.removeIf(f -> !filter.test("/" + f.getName()));
final String filename = archive.getId() + (snapshot == null ? "" : "-" + snapshot.getName()) + "." + ext;
String tag = snapshot.map(s -> "-" + s.getName()).orElse("");
final String filename = archive.getId() + tag + "." + ext;
ctx.header("Content-Type", ctype);
ctx.header("Content-Disposition", "attachment; filename=\"" + filename + "\"");
......@@ -273,8 +302,8 @@ public class ArchiveEndpoint implements RestBlueprint {
public Object handlePost(RestContext ctx) throws Exception {
throwIfHasSnapshot(ctx.getPathParam("archive"));
CDStarVault vault = SessionHelper.getCDStarSession(ctx, false).getVault(ctx.getPathParam("vault"));
CDStarArchive archive = openArchive(vault, ctx.getPathParam("archive"));
final CDStarVault vault = SessionHelper.getCDStarSession(ctx, false).getVault(ctx.getPathParam("vault"));
final CDStarArchive archive = openArchive(vault, ctx.getPathParam("archive"));
final QueryHelper qh = new QueryHelper(ctx);
final String action = qh.getExclusiveParam("trim", "snapshots");
......@@ -288,10 +317,27 @@ public class ArchiveEndpoint implements RestBlueprint {
return createOrUpdate(ctx, archive);
}
private SnapshotInfo createSnapshot(CDStarVault vault, CDStarArchive archive, RestContext ctx) throws IOException {
FormHelper form;
private SnapshotInfo createSnapshot(CDStarVault vault, CDStarArchive archive, RestContext ctx)
throws IOException, TARollbackException {
final FormHelper form = parseForm(ctx);
final String name = form.get("name")
.orElseThrow(() -> new ErrorResponse(400, "ApiError", "Missing form parameter")
.detail("field", "name"));
try {
ctx.status(201);
final SnapshotInfo info = snapshotInfo(archive.createSnapshot(name));
SessionHelper.commitOrSuspend(ctx);
return info;
} catch (final InvalidSnapshotName e) {
throw new ErrorResponse(400, "InvalidSnapshotName", "Invalid snapshot name");
}
}
static FormHelper parseForm(RestContext ctx) throws IOException {
try {
form = FormHelper.parse(ctx, 1024, 1024 * 8);
return FormHelper.parse(ctx, 1024, 1024 * 8);
} catch (final EntityTooLarge e) {
throw new ErrorResponse(413, "FormTooLarge", "Form request larger than expected.");
} catch (final UnsupportedContentType e) {
......@@ -301,20 +347,6 @@ public class ArchiveEndpoint implements RestBlueprint {
} catch (final FormParserException e) {
throw new ErrorResponse(400, "FormParserError", "Unable to parse form request.");
}
final String name = form.get("name")
.orElseThrow(() -> new ErrorResponse(400, "ApiError", "Missing form parameter").detail("field", "name"));
try {
final CDStarSnapshot snap = archive.createSnapshot(name);
final SnapshotInfo info = new SnapshotInfo();
info.name = snap.getName();
info.creator = snap.getCreator();
info.created = snap.getCreated();
return info;
} catch (final InvalidSnapshotName e) {
throw new ErrorResponse(400, "InvalidSnapshotName", "Invalid snapshot name");
}
}
/**
......@@ -323,8 +355,8 @@ public class ArchiveEndpoint implements RestBlueprint {
public Void handlePut(RestContext ctx) throws Exception {
throwIfHasSnapshot(ctx.getPathParam("archive"));
CDStarVault vault = SessionHelper.getCDStarSession(ctx, false).getVault(ctx.getPathParam("vault"));
CDStarArchive archive = openArchive(vault, ctx.getPathParam("archive"));
final CDStarVault vault = SessionHelper.getCDStarSession(ctx, false).getVault(ctx.getPathParam("vault"));
final CDStarArchive archive = openArchive(vault, ctx.getPathParam("archive"));
final QueryHelper qh = new QueryHelper(ctx);
final String subResource = qh.getExclusiveParam("acl", "meta");
......@@ -345,8 +377,8 @@ public class ArchiveEndpoint implements RestBlueprint {
public Void handleDelete(RestContext ctx) throws Exception {
CDStarVault vault = SessionHelper.getCDStarSession(ctx, false).getVault(ctx.getPathParam("vault"));
CDStarArchive archive = openArchive(vault, ctx.getPathParam("archive"));
final CDStarVault vault = SessionHelper.getCDStarSession(ctx, false).getVault(ctx.getPathParam("vault"));
final CDStarArchive archive = openArchive(vault, ctx.getPathParam("archive"));
final QueryHelper qh = new QueryHelper(ctx);
final String subResource = qh.getExclusiveParam("meta");
......@@ -361,12 +393,12 @@ public class ArchiveEndpoint implements RestBlueprint {
return null;
}
CDStarSnapshot snapshot = resolveSnapshot(archive, ctx.getPathParam("archive"));
final Optional<CDStarSnapshot> snapshot = resolveSnapshot(archive, ctx.getPathParam("archive"));
if (snapshot == null)
archive.remove();
if (snapshot.isPresent())
snapshot.get().remove();
else
snapshot.remove();
archive.remove();
// Remove entire archive
SessionHelper.commitOrSuspend(ctx);
......
......@@ -6,6 +6,7 @@ import java.nio.channels.WritableByteChannel;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import de.gwdg.cdstar.MimeUtils;
......@@ -49,13 +50,13 @@ public class FileEndpoint implements RestBlueprint {
* (?with=meta) or the MetaMap (?meta).
*/
public Object handleGet(RestContext ctx)
throws AccessError, ArchiveNotFound, VaultNotFound, IOException, SnapshotNotFound {
throws AccessError, ArchiveNotFound, VaultNotFound, IOException, SnapshotNotFound {
CDStarVault vault = SessionHelper.getCDStarSession(ctx, true).getVault(ctx.getPathParam("vault"));
CDStarArchive archive = ArchiveEndpoint.openArchive(vault, ctx.getPathParam("archive"));
CDStarSnapshot snapshot = ArchiveEndpoint.resolveSnapshot(archive, ctx.getPathParam("archive"));
CDStarFile file = snapshot == null ? archive.getFile(ctx.getPathParam("file"))
: snapshot.getFile(ctx.getPathParam("file"));
Optional<CDStarSnapshot> snapshot = ArchiveEndpoint.resolveSnapshot(archive, ctx.getPathParam("archive"));
String fileName = ctx.getPathParam("file");
CDStarFile file = snapshot.isPresent() ? snapshot.get().getFile(fileName) : archive.getFile(fileName);
final QueryHelper qh = new QueryHelper(ctx);
final String sub = qh.getExclusiveParam("info", "meta");
......@@ -126,7 +127,7 @@ public class FileEndpoint implements RestBlueprint {
* `If-None-Match: *` prevents overwriting existing files.
*/
public Void handlePut(RestContext ctx)
throws AccessError, ArchiveNotFound, VaultNotFound, FileNotFound, IOException, TARollbackException {
throws AccessError, ArchiveNotFound, VaultNotFound, FileNotFound, IOException, TARollbackException {
ArchiveEndpoint.throwIfHasSnapshot(ctx.getPathParam("archive"));
CDStarVault vault = SessionHelper.getCDStarSession(ctx, false).getVault(ctx.getPathParam("vault"));
......
package de.gwdg.cdstar.rest.v3;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.net.URISyntaxException;
import javax.ws.rs.client.Entity;
import javax.ws.rs.core.Form;
import javax.ws.rs.core.Response.Status;
import org.junit.Before;
import org.junit.Test;
import de.gwdg.cdstar.rest.BaseRestTest;
import de.gwdg.cdstar.web.common.model.SnapshotList;
public class SnapshotTest extends BaseRestTest {
private String archiveId;
@Before
public void prepareArchive() throws IOException, URISyntaxException {
target("/v3/test/").request().post(null);
assertStatus(Status.CREATED);
archiveId = getJsonString("id");
}
String makeSnapshot(String archiveId, String name) {
target("/v3/test/", archiveId)
.queryParam("snapshots", "").request()
.post(Entity.form(new Form("name", name)));
assertStatus(Status.CREATED);
assertEquals(name, getJsonString("name"));
assertEquals("test@system", getJsonString("creator"));
return archiveId + "@" + name;
}
String getFileAsString(String archiveId, String name) {
return target("/v3/test/", archiveId, name).request().get(String.class);
}
@Test
public void testListSnapshots() throws Exception {
final String v1 = makeSnapshot(archiveId, "v1");
final String v2 = makeSnapshot(archiveId, "v2");
final String v3 = makeSnapshot(archiveId, "v3");
final SnapshotList snapshots = target("/v3/test/" + archiveId).queryParam("snapshots", "").request()
.get(SnapshotList.class);
assertEquals(3, snapshots.getCount());
assertEquals(3, snapshots.getTotal());
assertEquals("v1", snapshots.getItems().get(0));
assertEquals("v2", snapshots.getItems().get(1));
assertEquals("v3", snapshots.getItems().get(2));
}
@Test
public void testBasicFunctionality() throws Exception {
target("/v3/test/", archiveId, "test.txt").request().post(Entity.text("v1"));
final String v1 = makeSnapshot(archiveId, "v1");
target("/v3/test/", archiveId, "test.txt").request().post(Entity.text("v2"));
target("/v3/test/", archiveId, "test2.txt").request().post(Entity.text("v2"));
final String v2 = makeSnapshot(archiveId, "v2");
target("/v3/test/", archiveId, "test.txt").request().delete();
final String v3 = makeSnapshot(archiveId, "v3");
target("/v3/test/" + archiveId);
}
}
......@@ -582,9 +582,11 @@ class ArchiveImpl implements CDStarArchive {
if (!name.matches("^[a-z0-9-.]+$"))
throw new InvalidSnapshotName(name, "Unsupported characters");
StorageObject snapSto = vault.createSTO();
final StorageObject snapSto = vault.createSTO();
try {
return snapshotList.createSnapshot(name, snapSto);
final CDStarSnapshot snap = getSnapshotsInternal().createSnapshot(name, snapSto);
markModified();
return snap;
} catch (InvalidSnapshotName | IOException | RuntimeException e) {
snapSto.remove();
if (e instanceof InvalidSnapshotName)
......
......@@ -175,14 +175,14 @@ class SnapshotImpl implements CDStarSnapshot {
public void remove() {
head.checkPermission(ArchivePermission.DELETE);
head.markModified();
sto.remove();
// Do NOT remove this snapshot from the list to prevent creating a new snapshot
// with the sane name.
load().remove();
// Do NOT remove this snapshot from the list to prevent creating a new
// snapshot with the sane name.
}
@Override
public boolean isRemoved() {
return sto.isRemoved();
return load().isRemoved();
}
}
package de.gwdg.cdstar.runtime;
import static org.junit.Assert.assertEquals;
import org.junit.Test;
import de.gwdg.cdstar.runtime.client.CDStarArchive;
public class SnapshotTest extends RuntimeBaseTest {
@Test
public void testArchiveInitialState() throws Exception {
CDStarArchive ar = makeArchive();
commit();
ar = loadArchive(ar.getId());
ar.createSnapshot("v1");