Commit a31018f8 authored by j.hoerdt's avatar j.hoerdt
Browse files

Merge branch '15-new-authentication-for-production-pid-reg' into 'master'

Resolve "new authentication for production PID reg"

Closes #2 and #15

See merge request !9
parents aaa9ed97 f87d1388
......@@ -22,7 +22,8 @@
!graphdb.gwdg.de.pem
!sensor_type_info.json
!sensor_info/**/*
!develogging.properties
!devel_sensor2graph.properties
......
......@@ -12,8 +12,12 @@ dependencies {
testImplementation 'junit:junit:4.12'
implementation 'org.neo4j.driver:neo4j-java-driver:4.2.4'
implementation 'com.google.code.gson:gson:2.8.6'
implementation 'com.squareup.okhttp3:okhttp:4.9.1'
}
compileJava {
options.compilerArgs << "-Xlint:deprecation"
}
mainClassName = 'sensor2graph.Main'
......
neo4j_uri=bolt://localhost:7687
database_name=neo4j
already_uploaded_days_file = all_days.txt
failed_to_upload_sensors_file = test_sensor.txt
\ No newline at end of file
# custom logging properties file based on default logging config file
############################################################
# Default Logging Configuration File
#
# You can use a different file by specifying a filename
# with the java.util.logging.config.file system property.
# For example java -Djava.util.logging.config.file=myfile
############################################################
############################################################
# Global properties
############################################################
# "handlers" specifies a comma separated list of log Handler
# classes. These handlers will be installed during VM startup.
# Note that these classes must be on the system classpath.
# By default we only configure a ConsoleHandler, which will only
# show messages at the INFO and above levels.
handlers=
# To also add the FileHandler, use the following line instead.
#handlers= java.util.logging.FileHandler, java.util.logging.ConsoleHandler
# Default global logging level.
# This specifies which kinds of events are logged across
# all loggers. For any given facility this global level
# can be overriden by a facility specific level
# Note that the ConsoleHandler also has a separate level
# setting to limit messages printed to the console.
.level= ALL
############################################################
# Handler specific properties.
# Describes specific configuration info for Handlers.
############################################################
# default file output is in user's home directory.
java.util.logging.FileHandler.pattern = logs/sensor%g_%u.log
java.util.logging.FileHandler.limit = 0
java.util.logging.FileHandler.count = 10
# Default number of locks FileHandler can obtain synchronously.
# This specifies maximum number of attempts to obtain lock file by FileHandler
# implemented by incrementing the unique field %u as per FileHandler API documentation.
java.util.logging.FileHandler.maxLocks = 100
java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter
# Limit the message that are printed on the console to INFO and above.
java.util.logging.ConsoleHandler.level = INFO
java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
# Example to customize the SimpleFormatter output format
# to print one-line log message like this:
# <level>: <log message> [<date/time>]
#
# java.util.logging.SimpleFormatter.format=%4$s: %5$s [%1$tc]%n
############################################################
# Facility specific properties.
# Provides extra control for each logger.
############################################################
# For example, set the com.xyz.foo logger to only log SEVERE
# messages:
# com.xyz.foo.level = SEVERE
sensor2graph.Main.level = FINE
sensor2graph.Main.handlers = java.util.logging.ConsoleHandler, java.util.logging.FileHandler
global.level = FINE
global.handlers = java.util.logging.FileHandler
......@@ -64,8 +64,8 @@ java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
# messages:
# com.xyz.foo.level = SEVERE
sensor2graph.Main.level = INFO
sensor2graph.Main.level = FINE
sensor2graph.Main.handlers = java.util.logging.ConsoleHandler, java.util.logging.FileHandler
global.level = INFO
global.level = FINE
global.handlers = java.util.logging.FileHandler
csv_directory = /mnt/nfs_root_data/luftdaten/
geocoding_cache = /mnt/nfs_root_data/luftdaten/geocoding_cache
already_uploaded_days_file = /progress/already_uploaded_days.txt
failed_to_upload_sensors_file = /progress/failed_to_upload.txt
\ No newline at end of file
failed_to_upload_sensors_file = /progress/failed_to_upload.txt
pid_reg_server_cert = /handle_auth/pid_reg_server_cert.pem
pid_reg_keystore = /handle_auth/pid_reg_keystore.p12
\ No newline at end of file
package sensor2graph;
import java.io.*;
import java.time.Duration;
import javax.net.ssl.*;
import java.security.*;
import java.security.cert.*;
import okhttp3.*;
public class Handle {
public static OkHttpClient http_client;
private static TrustManager[] get_trust_managers() throws Exception {
var server_self_signed_certificate = CertificateFactory.getInstance("X.509")
.generateCertificate(new FileInputStream(Main.pid_reg_server_cert));
var server_cert_public_key_store = KeyStore.getInstance(KeyStore.getDefaultType());
server_cert_public_key_store.load(null, null);
server_cert_public_key_store.setCertificateEntry("vm13.pid.gwdg.de", server_self_signed_certificate);
var trust_manager_factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
trust_manager_factory.init(server_cert_public_key_store);
return trust_manager_factory.getTrustManagers();
}
private static KeyManager[] get_key_managers() throws Exception {
var private_key_store = KeyStore.getInstance(KeyStore.getDefaultType());
private_key_store.load(new FileInputStream(Main.pid_reg_server_cert), "asdfasdf".toCharArray());
var key_manager_factory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
key_manager_factory.init(private_key_store, "asdfasdf".toCharArray());
return key_manager_factory.getKeyManagers();
}
private static HostnameVerifier allowAllHostNames() {
return (hostname, sslSession) -> true;
}
public static OkHttpClient get_client() throws Exception, NoSuchAlgorithmException, KeyManagementException {
var trust_managers = get_trust_managers();
var trust_self_signed_cert_and_provide_client_cert = SSLContext.getInstance("TLS");
trust_self_signed_cert_and_provide_client_cert.init(get_key_managers(), trust_managers, null);
var client = new OkHttpClient.Builder()
.sslSocketFactory(trust_self_signed_cert_and_provide_client_cert.getSocketFactory(), (X509TrustManager) trust_managers[0])
.hostnameVerifier(allowAllHostNames())
.connectTimeout(Duration.ofSeconds(20))
.build();
return client;
}
}
......@@ -34,9 +34,12 @@ public class Main {
private static String archive_uri;
public static String pid_registry_uri;
public static String prefix_for_new_pids;
public static String pid_reg_server_cert;
public static String pid_reg_keystore;
// }
public static Driver driver;
public static String handle_registry_session_id;
public static java.util.logging.Logger glogger;
private static java.util.logging.Logger logger;
static {
......@@ -56,8 +59,8 @@ public class Main {
//.withMaxTransactionRetryTime(3, TimeUnit.SECONDS)
//.withLeakedSessionsLogging()
.withMaxConnectionPoolSize(concurrency)
// .withEncryption()
// .withTrustStrategy(TrustStrategy.trustCustomCertificateSignedBy(neo4j_server_certificate_file))
.withEncryption()
.withTrustStrategy(TrustStrategy.trustCustomCertificateSignedBy(neo4j_server_certificate_file))
.build()
);
}
......@@ -78,8 +81,8 @@ public class Main {
private static void run() throws Exception {
try_uploading_failed_sensors();
upload_all_days();
try_uploading_failed_sensors();
// upload_all_days();
// try_uploading_failed_sensors();
}
......@@ -163,10 +166,13 @@ public class Main {
Main.logger.fine("initializing neo4j driver.");
initialize_driver();
try {
Handle.http_client = Handle.get_client();
test_auth();
pre_run_graphdb_actions();
//run();
run();
} catch (org.neo4j.driver.exceptions.AuthenticationException e) {
Main.logger.log(Level.SEVERE, "Authentication failed, are your credentials correct?");
} catch (Exception e) {
......@@ -192,6 +198,8 @@ public class Main {
archive_uri = config.getProperty("archive_uri" );
pid_registry_uri = config.getProperty("pid_registry_uri" );
prefix_for_new_pids = config.getProperty("prefix_for_new_pids" );
pid_reg_server_cert = config.getProperty("pid_reg_server_cert" );
pid_reg_keystore = config.getProperty("pid_reg_keystore" );
}
......@@ -208,18 +216,20 @@ public class Main {
private static Properties get_default_properties() {
Properties defaults = new Properties();
defaults.setProperty("already_uploaded_days_file", "already_uploaded_days.txt" );
defaults.setProperty("failed_to_upload_sensors_file", "failed_to_upload.txt" );
defaults.setProperty("database_name", "sensor" );
defaults.setProperty("csv_directory", "data/csv_files/" );
defaults.setProperty("geocoding_cache", "data/geocoding_cache/" );
defaults.setProperty("concurrency", "50" );
defaults.setProperty("http_agent", "Please/0.5" );
defaults.setProperty("neo4j_uri", "bolt://graphdb.gwdg.de:7687" );
defaults.setProperty("neo4j_server_certificate_file", "graphdb.gwdg.de.pem" );
defaults.setProperty("archive_uri", "https://archive.sensor.community/" );
defaults.setProperty("pid_registry_uri", "http://vm04.pid.gwdg.de:8081/handles/");
defaults.setProperty("prefix_for_new_pids", "21.T11998/" );
defaults.setProperty("already_uploaded_days_file", "already_uploaded_days.txt" );
defaults.setProperty("failed_to_upload_sensors_file", "failed_to_upload.txt" );
defaults.setProperty("database_name", "sensor" );
defaults.setProperty("csv_directory", "data/csv_files/" );
defaults.setProperty("geocoding_cache", "data/geocoding_cache/" );
defaults.setProperty("concurrency", "1" );
defaults.setProperty("http_agent", "Please/0.5" );
defaults.setProperty("neo4j_uri", "bolt://graphdb.gwdg.de:7687" );
defaults.setProperty("neo4j_server_certificate_file", "graphdb.gwdg.de.pem" );
defaults.setProperty("archive_uri", "https://archive.sensor.community/" );
defaults.setProperty("pid_registry_uri", "https://vm13.pid.gwdg.de:8000/api/handles/");
defaults.setProperty("prefix_for_new_pids", "21.11138/" );
defaults.setProperty("pid_reg_server_cert", "pid_reg_server_cert.pem" );
defaults.setProperty("pid_reg_keystore", "pid_reg_keystore.p12" );
return defaults;
}
......
......@@ -5,26 +5,16 @@ import sensor2graph.georev.*;
import org.neo4j.driver.*;
import org.neo4j.driver.Record;
import okhttp3.*;
import com.google.gson.*;
import java.net.*;
import java.net.http.*;
import java.net.http.HttpRequest.BodyPublishers;
import java.net.http.HttpResponse.BodyHandlers;
import java.time.Duration;
import java.util.*;
import java.util.function.*;
import java.util.logging.*;
import java.util.regex.*;
class Sensor {
private static HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(20))
.authenticator(new Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(System.getenv("SENSOR2GRAPH_PIDREG_USER"), System.getenv("SENSOR2GRAPH_PIDREG_PASS").toCharArray());
}
}).build();
private Map<String, Object> properties = new HashMap<String, Object>();
public void process(Session session, TransactionConfig transaction_config) throws Exception {
......@@ -52,8 +42,10 @@ class Sensor {
}
private String register_new_pid(Record rec) throws Exception {
String response_body = request_to_registry("POST", Main.prefix_for_new_pids, rec);
return new Gson().fromJson(response_body, JsonObject.class).get("epic-pid").getAsString();
String response_body = request_to_registry("PUT", Main.prefix_for_new_pids + UUID.randomUUID().toString(), rec);
var created_pid = new Gson().fromJson(response_body, JsonObject.class).get("handle").getAsString();
Main.glogger.info("created pid: " + created_pid);
return created_pid;
}
private void update_pid(String pid, Record rec) throws Exception {
......@@ -61,45 +53,63 @@ class Sensor {
}
private String request_to_registry(String method, String target_uri, Record rec) throws Exception {
JsonArray body = new JsonArray();
add_attribute(body,
"21.T11148/eb3c713572f681e6c4c3",
"[{\"AlternateIdentifier\":{\"AlternateIdentifierValue\":\"" + properties.get("sensor_id") + "\",\"alternateIdentifierType\":\"sensor_id\"}}]"
);
var body = new JsonObject();
body.add("values", new JsonArray());
{
var admin_data = new JsonObject();
var admin_data_value = new JsonObject();
admin_data_value.addProperty("index", "300");
admin_data_value.addProperty("handle", "21.11138/USER01");
admin_data_value.addProperty("permissions", "110011111111");
admin_data.addProperty("format", "admin");
admin_data.add("value", admin_data_value);
add_attribute(body, "100", "HS_ADMIN", admin_data);
}
if (!rec.get("s.pid").isNull()) {
add_attribute(body,
"1",
"21.T11148/8eb858ee0b12e8e463a5",
new JsonPrimitive("{\"identifier-general-with-type\":{\"identifierValue\":\""+ rec.get("s.pid") +"\", \"identifierType\":\"Handle\"}}")
);
}
add_attribute(body,
"2",
"21.T11148/22c62082a4d2d9ae2602",
"[{\"dateType\":\"Commissioned\", \"date\":\"" + rec.get("min(all_measurements.first_msg)").asLocalDate().toString() + "\"},{\"dateType\":\"last message\", \"date\":\"" + rec.get("max(all_measurements.last_msg)").asLocalDate().toString() + "\"}]"
new JsonPrimitive("[{\"dateType\":\"Commissioned\", \"date\":\"" + rec.get("min(all_measurements.first_msg)").asLocalDate().toString() + "\"},{\"dateType\":\"last message\", \"date\":\"" + rec.get("max(all_measurements.last_msg)").asLocalDate().toString() + "\"}]")
);
add_attribute(body,
"3",
"21.T11148/709a23220f2c3d64d1e1",
'\"' + rec.get("sensor_type").asString() + '\"'
new JsonPrimitive('\"' + rec.get("s.sensor_id").asString() + '\"')
);
add_attribute(body,
"4",
"21.T11148/68c2cc7f0ceaa3e499ec",
"{\"Model\":{\"modelName\":\"" + rec.get("sensor_type").asString() + "\"}}"
new JsonPrimitive("{\"Model\":{\"modelName\":\"" + rec.get("sensor_type").asString() + "\"}}")
);
add_attribute(body,
"5",
"21.T11148/9a15a4735d4bda329d80",
"\"https://sensor.community\""
new JsonPrimitive("\"https://sensor.community\"")
);
add_attribute(body,
"6",
"21.T11148/4eaec4bc0f1df68ab2a7",
"{\"Owners\":[{\"Owner\":{\"ownerName\":\"SensorCommunity\",\"ownerContact\":\"contact@open-forecast.eu\"}}]}"
new JsonPrimitive("{\"Owners\":[{\"Owner\":{\"ownerName\":\"SensorCommunity\",\"ownerContact\":\"contact@open-forecast.eu\"}}]}")
);
add_attribute(body,
"7",
"21.T11148/1f3e82ddf0697a497432",
"{\"Manufacturers\":[{\"Manufacturer\":{\"manufacturerName\":\""+ rec.get("manufacturer").asString() +"\"}}]}"
new JsonPrimitive("{\"Manufacturers\":[{\"Manufacturer\":{\"manufacturerName\":\""+ rec.get("manufacturer").asString() +"\"}}]}")
);
if (!rec.get("description").isNull()) {
add_attribute(body,
"8",
"21.T11148/55f8ebc805e65b5b71dd",
"\""+ rec.get("description") +"\""
);
}
if (!rec.get("s.pid").isNull()) {
add_attribute(body,
"21.T11148/8eb858ee0b12e8e463a5",
"{\"identifier-general-with-type\":{\"identifierValue\":\""+ rec.get("s.pid") +"\", \"identifierType\":\"Handle\"}}"
new JsonPrimitive("\""+ rec.get("description") +"\"")
);
}
// add_attribute(body,
......@@ -111,27 +121,36 @@ class Sensor {
Main.glogger.fine("sent body: " + body);
HttpRequest request = HttpRequest.newBuilder(URI.create(Main.pid_registry_uri + target_uri))
.header("Content-Type", "application/json")
Request request = new Request.Builder()
.url(Main.pid_registry_uri + target_uri)
.method(method, RequestBody.create(body.toString(), MediaType.parse("application/json")))
.header("Accept", "application/json")
.method(method, BodyPublishers.ofString(body.toString())).build();
.header("Content-Type", "application/json")
.header("Authorization", "Handle clientCert=\"true\"")
.build();
HttpResponse<String> response = client.send(request, BodyHandlers.ofString());
Main.glogger.fine("status: " + response.statusCode());
Main.glogger.fine("resp: " + response.body());
Main.glogger.fine("request uri: " + Main.pid_registry_uri + target_uri);
if (response.statusCode() / 100 != 2) {
throw new Exception("request to pid registry failed with status code " + response.statusCode() + ", reponse: " + response.body());
}
try (Response response = Handle.http_client.newCall(request).execute()) {
var response_body = response.body().string();
Main.glogger.fine("status: " + response.code());
Main.glogger.fine("resp: " + response_body);
return response.body();
if (response.code() / 100 != 2) {
throw new Exception("request to pid registry failed with status code " + response.code() + ", reponse: " + response_body);
}
return response_body;
}
}
private void add_attribute(JsonArray body, String registered_type, String parsed_data) {
private void add_attribute(JsonObject body, String index, String type, JsonElement data) {
JsonObject obj = new JsonObject();
obj.addProperty("type", registered_type);
obj.addProperty("parsed_data", parsed_data);
body.add(obj);
obj.addProperty("index", index);
obj.addProperty("type", type);
obj.add("data", data);
body.getAsJsonArray("values").add(obj);
}
private Query get_creation_query() {
......@@ -174,7 +193,7 @@ class Sensor {
"with s, type.sensor_type as sensor_type, type.manufacturer as manufacturer, type.description as description\n" +
"match (s)--(all_measurements:MeasurementLocation)" +
"return s.pid, id(s), sensor_type, manufacturer, description min(all_measurements.first_msg), max(all_measurements.last_msg)"
"return s.pid, id(s), s.sensor_id, sensor_type, manufacturer, description, min(all_measurements.first_msg), max(all_measurements.last_msg)"
).withParameters(properties);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment