Commit 013a3756 authored by mhellka's avatar mhellka
Browse files

Drastically reduced NioPool default cache size and saved some heap.

parent 39a595ac
Pipeline #117125 passed with stages
in 9 minutes and 32 seconds
......@@ -64,7 +64,7 @@ public class NioPool implements StoragePool {
public static final int SHARD_DEPTH = 2;
private static JsonFormat objectLoader = new JsonFormat();
private int cacheSize = 1024;
private int cacheSize = 16;
// Running transactions
ConcurrentHashMap<String, NioSession> scopes = new ConcurrentHashMap<>();
......
......@@ -120,16 +120,15 @@ public class JsonFormat {
case ("resources"):
assertNextToken(p, JsonToken.START_ARRAY);
doc.resources = new ArrayList<>();
while (p.nextToken() == JsonToken.START_OBJECT) {
while (p.nextToken() == JsonToken.START_OBJECT)
doc.resources.add(readResourceEntry(p));
}
assertCurrentToken(p, JsonToken.END_ARRAY);
break;
default:
if (fieldName.startsWith("x-")) {
if (doc.attr == null)
doc.attr = new HashMap<>(4);
doc.attr.put(fieldName.substring(2).intern(), readText(p));
doc.attr.put(fieldName.substring(2), readText(p));
} else {
throw new FormatError("Unknown property: " + fieldName, p);
}
......@@ -150,9 +149,47 @@ public class JsonFormat {
assertCurrentToken(p, JsonToken.END_OBJECT);
compress(doc);
return doc;
}
/**
* Try to reduce document size by deduplicating certain strings.
*/
private void compress(JsonIndex doc) {
// Resources often share the same 'type' and 'enc' values. Frozen objects likely
// have the same 'src' value for all resources. To speed things up and not
// produce more garbage than we save, we only compare consecutive resource here.
String lastType = null;
String lastSrc = null;
String lastEnc = null;
for (final JsonResource res : doc.resources) {
if (res.type != null) {
if (res.type.equals(lastType))
res.type = lastType;
else
lastType = res.type;
}
if (res.src != null) {
if (res.src.equals(lastSrc))
res.src = lastSrc;
else
lastSrc = res.src;
}
if (res.enc != null) {
if (res.enc.equals(lastSrc))
res.enc = lastEnc;
else
lastEnc = res.enc;
}
}
}
private JsonResource readResourceEntry(JsonParser p) throws IOException, FormatError {
final JsonResource jr = new JsonResource();
jr.size = -1;
......@@ -195,7 +232,7 @@ public class JsonFormat {
if (fieldName.startsWith("x-")) {
if (jr.attr == null)
jr.attr = new HashMap<>(4);
jr.attr.put(fieldName.substring(2).intern(), readText(p));
jr.attr.put(fieldName.substring(2), readText(p));
} else {
throw new FormatError("Unknown property: " + fieldName, p);
}
......@@ -225,7 +262,8 @@ public class JsonFormat {
}
private byte[] readBytes(JsonParser p) throws IOException, FormatError {
return Utils.base64decode(readText(p));
assertNextToken(p, JsonToken.VALUE_STRING);
return p.getBinaryValue();
}
/*
......@@ -400,7 +438,7 @@ public class JsonFormat {
if (src.startsWith("external:")) {
res.put("src", "x-" + src.substring("external:".length()));
} else if (src.endsWith(".bin")
&& src.equals(Utils.bytesToHex(Utils.base64decode(res.get("sha256").asText())) + ".bin")) {
&& src.equals(Utils.bytesToHex(Utils.base64decode(res.get("sha256").asText())) + ".bin")) {
res.remove("src");
} else {
throw new FormatError("Unrecognized value for 'resource.src' in version 3 format: " + Utils.repr(src));
......
......@@ -9,9 +9,8 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include;
@JsonInclude(Include.NON_NULL)
public final class JsonResource {
public String id; // Unique tracking ID that never changes
public String src; // Source filename (relative to object path), null, or
// prefixed URI
public String name; // Given name of this file
public String src; // Null or prefixed URI
public String name; // Given name for this file
public String type; // Content-Type compatible mime type
public String enc; // Content-Encoding compatible encoding name. May be
// null if no encoding is applied.
......
......@@ -20,6 +20,7 @@ import de.gwdg.cdstar.runtime.client.exc.InvalidFileName;
class FileImpl implements CDStarFile {
private static final String DEFAULT_MEDIA_TYPE = "application/octet-stream";
static final String RESOURCE_PREFIX = "data/";
final ArchiveImpl archive;
final Resource resource;
......@@ -139,7 +140,7 @@ class FileImpl implements CDStarFile {
public String getMediaType() {
final String type = resource.getMediaType();
if (type == null)
return "application/octet-stream";
return DEFAULT_MEDIA_TYPE;
return type;
}
......@@ -155,6 +156,8 @@ class FileImpl implements CDStarFile {
archive.checkPermission(ArchivePermission.CHANGE_FILES);
if (Utils.equal(getMediaType(), type) && Utils.equal(getContentEncoding(), coding))
return;
if (Utils.equal(type, DEFAULT_MEDIA_TYPE))
type = null; // Do not store the obvious
resource.setMediaType(type, coding);
archive.markContentModified();
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment