Commit 903d90b4 authored by malzer's avatar malzer
Browse files

Fix issue #51 by providing an option to import KML as CSV

parent e26e0f33
......@@ -16,6 +16,15 @@
padding-bottom: 5px;
}
.console-large {
background-color: #fff;
color: #888;
font-size: 120%;
white-space: normal;
line-height: 130%;
padding-bottom: 5px;
}
.consoleNoBg {
color: #888;
font-size: 80%;
......
......@@ -12,7 +12,7 @@
<link rel="stylesheet" type="text/css" media="screen" href="https://res.de.dariah.eu/dhrep/css/bootstrap-customization.css" />
<link rel="stylesheet" type="text/css" media="screen" href="https://res.de.dariah.eu/dhrep/css/bootstrap-modal.css" />
<link rel="stylesheet" type="text/css" media="screen" href="https://res.de.dariah.eu/dhrep/css/font-awesome.css" />
<link rel="stylesheet" media="screen" href="https://cdn.jsdelivr.net/npm/handsontable@9.0.0/dist/handsontable.full.min.css">
<link rel="stylesheet" media="screen" href="https://cdn.jsdelivr.net/npm/handsontable@9.0.1/dist/handsontable.full.min.css">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/openlayers/openlayers.github.io@master/en/v6.5.0/css/ol.css" type="text/css">
<link rel="stylesheet" href="https://dariah-de.github.io/status/dariah/embed.css">
<link rel="stylesheet" type="text/css" media="screen" href="./css/dariah.workflow.css" />
......@@ -119,7 +119,12 @@
<div class="alert alert-info alert-block">
<p><strong><a href="https://geobrowser.de.dariah.eu/doc/datasheet.html" target="_blank">What's new in version 3.6?</a></strong> &nbsp; ...more: <strong><a href="https://geobrowser.de.dariah.eu/doc/datasheet.html#gui-documentation" target="_blank">GUI documentation – how does it work?</a></strong></p>
</div>
<div id="alertArea"></div>
<div id="alertArea">
<p>
<img id="importDataSpinner" class="hide" src="https://res.de.dariah.eu/dhrep/img/spinning-flower_slow.gif" alt="Importing data..." width="22" />
<span id="importMessage" class="console-large">&nbsp;</span>
</p>
</div>
<div id="sheetArea">
<div id="howtoArea"></div>
<button id="howtoButton" class="btn btn-small pull-right" onclick="toggleHowto();" title="Read a small howto to get help filling in the table data.">How to fill the table</button>
......@@ -188,7 +193,7 @@
</button>
<p></p>
<button id="importButton" class="btn btn-primary" style="width:100%;" onclick="toggleUploadButton()" title="Create a new datasheet by importing a local CSV file.">
<i class="icon-upload"></i> &nbsp;Import local CSV file</button>
<i class="icon-upload"></i> &nbsp;Import local CSV or KML file</button>
<form id="uploadButton" style="display:none;">
<input id="files" class="btn btn-small" type="file" name="files[]" style="width:91%;"/>
</form>
......@@ -331,7 +336,7 @@
<script src="https://code.jquery.com/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/jquery-csv@1.0.21/src/jquery.csv.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/underscore@1.13.1/underscore-umd-min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/handsontable@9.0.0/dist/handsontable.full.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/handsontable@9.0.1/dist/handsontable.full.min.js"></script>
<script src="https://res.de.dariah.eu/dhrep/js/bootstrap.js"></script>
<!-- OpenLayers -->
<script src="https://cdn.polyfill.io/v2/polyfill.min.js?features=requestAnimationFrame,Element.prototype.classList"></script>
......
......@@ -24,6 +24,7 @@ function postToDariahStorage(postdata) {
// Disable create and import buttons to prevent double-clicking on them.
$('#newButton').addClass('disabled');
$('#importButton').addClass('disabled');
$(".fileImportError").alert('close');
// Create new file.
$.ajax({
url: storageURL,
......
......@@ -18,7 +18,7 @@ const requiredColumnsGeobrowser = ["Address", "TimeStamp", "TimeSpan:begin", "Ti
// columns with date format
const dateColumns = ["TimeStamp", "TimeSpan:begin", "TimeSpan:end"];
// Mimetypes which pass check to get parsed into table.
var allowedMimeTypes = ["text/csv", "text/comma-separated-values", "application/vnd.dariahde.geobrowser.csv", "application/vnd.ms-excel"]
var allowedMimeTypes = ["text/csv", "text/comma-separated-values", "application/vnd.dariahde.geobrowser.csv", "application/vnd.ms-excel", "application/vnd.google-earth.kml+xml", "text/xml"]
// CSV mimetype used for storing datasheets.
var csvStorageMimetype = 'application/vnd.dariahde.geobrowser.csv';
......
......@@ -29,13 +29,13 @@ var dateTimeSchemaUrl = 'http://books.xmlschemata.org/relaxng/ch19-77049.html';
if(rowIndex !== 0) {
var rvd1 = isValidDate(rowIndex, rowData, timeStampCol);
result = rvd1 && result;
if (!rvd1) errorInRow.push(errLink(rowIndex + 1, timeStampCol));
if (!rvd1) errorInRow.push(errLink(rowIndex, timeStampCol));
var rvd2 = isValidDate(rowIndex, rowData, timeSpanBCol);
result = rvd2 && result;
if (!rvd2) errorInRow.push(errLink(rowIndex + 1, timeSpanCol));
if (!rvd2) errorInRow.push(errLink(rowIndex, timeSpanCol));
var rvd3 = isValidDate(rowIndex, rowData, timeSpanECol);
result = rvd3 && result;
if (!rvd3) errorInRow.push(errLink(rowIndex + 1, timeSpanECol));
if (!rvd3) errorInRow.push(errLink(rowIndex, timeSpanECol));
// Allow empty time fields by commenting out the following:
//if (!(rvd1 && rvd2 && rvd3)) errorInRow.push(rowIndex + 1);
}
......
......@@ -172,7 +172,7 @@ function createNewDataset() {
// Fill with initial data.
var initialData = createInitialTableContent();
tableInstance.loadData(initialData);
setTableReadOnly(false);
if (tableInstance.getSettings().readOnly) setTableReadOnly(false);
// Create CSV structure.
var arr = arr2csv(initialData);
// Create new data file.
......@@ -280,19 +280,23 @@ function loadFromFiles(files) {
//console.log(f.name + ' - ' + f.type || 'n/a' + ' - ' + f.size + ' bytes, last modified: ' + f.lastModifiedDate.toLocaleDateString());
// Check for mimetype (browser-based).
if($.inArray(f.type, allowedMimeTypes) < 0 ) {
newAlert('error', 'Could not open file with name “' + f.name + '“!', 'Maybe the mimetype “' + f.type + '“ is not supported, can not be determined or your file is not a CSV file at all. Datasheet Editor can only handle CSV files of the following mimetypes: <strong>' + allowedMimeTypes + '</strong>. If your file really is a CSV file, please try adding “.csv“ as a file suffix, so your browser can determine the mimetype.', '');
newAlert('error', 'Could not open file with name “' + f.name + '“!', 'Maybe the mimetype “' + f.type + '“ is not supported, can not be determined or your file is not a CSV or KML file at all. Datasheet Editor can only handle files of the following mimetypes: <strong>' + allowedMimeTypes + '</strong>. If your file really is a CSV or KML file, please try adding “.csv“ or “.kml“ as a file suffix, so your browser can determine the mimetype.', 'fileImportError');
continue;
}
var isKML = f.type === "application/vnd.google-earth.kml+xml" || f.type === "text/xml";
if (isKML) setImportMessage('Converting KML to CSV...', true);
else setImportMessage('Importing CSV...', true);
var reader = new FileReader();
reader.onload = function(e) {
var data = e.target.result;
// TODO Check for allowed text snippets to determine if a loaded file may be a CSV file?
// Validate file to be a valid CSV file here?
if (isKML) data = loadKMLFromCSV(data);
setImportMessage('', false);
if (readToken() !== null) {
if (csvToTable(data)) postToDariahStorage(data);
if (isKML && data) postToDariahStorage(data);
else if (!isKML && csvToTable(data)) postToDariahStorage(data);
} else {
// Cache data.
setFiles(data);
if (data) setFiles(data);
authenticate('action=create-files');
}
};
......@@ -300,6 +304,54 @@ function loadFromFiles(files) {
}
}
function getXMLTag(root, tag){
//double quotes not allowed because they are used as delimiters
var element = $(root).find(tag).text();
return element.replaceAll('"',"'");
}
function loadKMLFromCSV(data){
$(".fileImportSuccess").alert('close');
$xml = '';
try {
$xml = $($.parseXML(data));
} catch (e){
newAlert('error', 'Error reading KML!', '<p>Could not read local KML file! Please make sure that the file corresponds to the <a target="_blank" href="https://geobrowser.de.dariah.eu/doc/geobrowser.html#specification-for-use">specification</a>.<br>Error message: '+e.message+'</p>', 'fileImportError');
return false;
}
var placemarks = $xml.find( "Placemark" );
if (!placemarks || isEmptyString(placemarks.text())) {
newAlert('error', 'Error reading KML!', '<p>Could not read local KML file! "Placemark" tag is missing. Please make sure that the file corresponds to the <a target="_blank" href="https://geobrowser.de.dariah.eu/doc/geobrowser.html#specification-for-use">specification</a>.</p>', 'fileImportError');
return false;
}
var rows = [["Name", "Address", "Description", "Longitude", "Latitude", "TimeStamp", "TimeSpan:begin", "TimeSpan:end", "GettyID"]]; // do not use globals here!
var name, address, description, point, coords, lon, lat, timeStamps, timeStamp, timeStampBegin, timeStampEnd, row;
$.each(placemarks, function(index, placemark){
name = getXMLTag(placemark, "name");
address = getXMLTag(placemark, "address");
description = getXMLTag(placemark, "description");
point = $(placemark).find("Point");
coords = getXMLTag(point, 'coordinates');
lon = coords.split(',')[0];
lat = coords.split(',')[1];
timeStamps = $(placemark).find("TimeStamp");
timeStamp = getXMLTag(timeStamps, "when");
timeStampBegin = getXMLTag(timeStamps, "begin");
timeStampEnd = getXMLTag(timeStamps, "end");
row = [name, address, description, lon, lat, timeStamp, timeStampBegin, timeStampEnd, ''];
rows.push(row);
})
setTableReadOnly(false);
hideFirstStartWizard();
newAlert('success', 'Converted KML to CSV!', '<p>Successfully converted KML to CSV. Default CSV column headers have been added. Please note that only following tags from the <a target="_blank" href="https://geobrowser.de.dariah.eu/doc/geobrowser.html#specification-for-use">KML specification</a> are considered during import: <i>Placemark, name, address, description, Point (coordinates), TimeStamp (when/begin/end)</i>. All other tags are ignored by default! </p>', 'fileImportSuccess');
tableInstance.loadData(rows);
tableInstance.validateCells();
tableInstance.render();
return arr2csv(tableInstance.getData());
}
/**
*
*/
......@@ -307,12 +359,12 @@ function csvToTable(data) {
setTableReadOnly(false);
var success = true;
try {
var arr = $.csv.toArrays(data);
var arr = $.csv.toArrays(data); //default delimiter: "
tableInstance.loadData(arr);
hideFirstStartWizard();
} catch (e){
success = false;
newAlert('error', 'Error reading CSV!', '<p>Could not read local CSV file! Maybe illegal format / illegal characters?</p>');
newAlert('error', 'Error reading CSV!', '<p>Could not read local CSV file! Error message: <i>'+e.message+'</i></p>', 'fileImportError');
}
return success;
}
......@@ -320,9 +372,7 @@ function csvToTable(data) {
function setTableReadOnly(readOnly){
tableInstance.updateSettings({
cells: function (row, col) {
return handleCells(row, col, readOnly);
}
readOnly: readOnly
});
}
/**
......@@ -330,9 +380,9 @@ function setTableReadOnly(readOnly){
*/
function csvToReadOnlyTable(data) {
setTableReadOnly(true);
hideFirstStartWizard();
var arr = $.csv.toArrays(data);
tableInstance.loadData(arr);
hideFirstStartWizard();
setPermanentConsoleMessage('This dataset is read-only.');
showMessage('idMessage', 'This dataset was loaded from the old DARIAH-DE Storage and has got the ID');
showMessage('sharedStatus', 'This dataset is public.');
......@@ -484,6 +534,7 @@ function doThingsOnIDExistingCheck () {
$('#dataLoc button').addClass('disabled');
disableManagedButtons();
}
if (tableInstance) tableInstance.render(); //re-render after DOM updates
}
/**
......@@ -700,6 +751,12 @@ function logWidth() {
console.log("dt owi: " + $('#dataTable').outerWidth(true));
}
function setImportMessage (message, show) {
if (show) $('#importDataSpinner').removeClass('hide');
else $('#importDataSpinner').addClass('hide');
showMessage('importMessage', message);
}
/**
* Show console message and spin the flower!
*/
......
......@@ -6,8 +6,8 @@
*/
function createInitialTableContent(){
// column titles are placed in the first row, i.e. are no actual column headers
var initialData = [defaultColumnHeaders];
// column titles are placed in the first row, i.e. are no actual column headers. Do not use globals here!
var initialData = [["Name", "Address", "Description", "Longitude", "Latitude", "TimeStamp", "TimeSpan:begin", "TimeSpan:end", "GettyID"]];
for (var i = 0; i < 30; i++) initialData.push(['', '', '', '', '', '', '', '', '']);
return initialData;
}
......@@ -16,29 +16,36 @@
* Handles cell settings, i.e. read-only, autocomplete and date format
*/
function handleCells(row, col, readOnly) {
if (!readOnly) readOnly = false;
var cellProperties = {readOnly: readOnly};
function handleHeaderCells(row) {
if (row === 0) { //header
cellProperties.renderer = headerRenderer;
cellProperties.type = 'autocomplete';
cellProperties.trimDropdown = false;
cellProperties.source = defaultColumnHeaders;
cellProperties.strict = false;
} else {
var dateColumns = getDateColumns();
if (dateColumns && dateColumns.includes(col)) {
cellProperties.type = 'date';
cellProperties.correctFormat = true;
cellProperties.dateFormat = 'YYYY-MM-DD';
cellProperties.defaultDate = '1900-01-01';
cellProperties.datePickerConfig = {
yearRange: [1000, 2050]
}
} else cellProperties.type = 'text';
return {
renderer: headerRenderer,
type: 'autocomplete',
trimDropdown: false,
source: ["Name", "Address", "Description", "Longitude", "Latitude", "TimeStamp", "TimeSpan:begin", "TimeSpan:end", "GettyID"],
strict: false,
};
}
return cellProperties;
}
/**
* Returns settings for custom date type field
*/
function getDateFieldProps(){
return {
validator: function(value, callback){ //custom date validator
if (xsdGYear.test(value) || xsdGYearMonth.test(value) || xsdDateOrDateTime.test(value)) {
callback(true);
} else callback(false);
},
type: 'date',
correctFormat: false,
dateFormat: 'YYYY-MM-DD',
defaultDate: '1900-01-01',
datePickerConfig: {
yearRange: [0, 2050]
}
};
}
function initTable() {
......@@ -71,29 +78,41 @@ function initTable() {
outsideClickDeselects: function (event) {
//clicking buttons does not clear selection
return !(event.nodeName === 'BUTTON' || event.parentElement && event.parentElement.nodeName === 'BUTTON');
},
minSpareCols: 1, // always keep at least 1 spare column at the right
minSpareCols: 1, // always keep at least 1 spare row at the right
minSpareRows: 1, // always keep at least 1 spare row at the bottom
cells: handleCells,
cells: handleHeaderCells, // handles header row
afterLoadData: function(){ //set date formats
var dateColumnsIndices = getDateColumns();
if (dateColumnsIndices) {
var dateProps = getDateFieldProps();
$.each(dateColumnsIndices, function (index, column) {
for (var i = 1; i < tableInstance.getDataAtCol(column).length; i++) {
tableInstance.setCellMetaObject(i, column, dateProps);
}
});
tableInstance.render();
}
},
afterChange: function (changes, source) {
if (!changes) return;
if (source !== 'loadData') { //don't save on load
autoSave(changes);
}
changes.forEach(function([row, prop, oldValue, newValue]) {
if (row === 0){
var readOnly = tableInstance.getCellMeta(0, 0).readOnly;
if (dateColumns.includes(newValue) || (dateColumns.includes(oldValue) && !dateColumns.includes(newValue))){
tableInstance.updateSettings({ //re-render cells
cells: function (row, col) {
return handleCells(row, col, readOnly);
}
});
changes.forEach(function([row, col, oldValue, newValue]) {
if (row === 0){ //update columns with date format
var addedDateColumn = dateColumns.includes(newValue);
var removedDateColumn = dateColumns.includes(oldValue) && !dateColumns.includes(newValue);
if (addedDateColumn || removedDateColumn){
var dateProps = getDateFieldProps();
for (var i = 1; i<tableInstance.getDataAtCol(col).length; i++) {
if (addedDateColumn) tableInstance.setCellMetaObject(i, col, dateProps);
else if (removedDateColumn) tableInstance.setCellMetaObject(i, col, {"type": "text", validator: null});
}
tableInstance.render();
}
}
});
}
});
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment