Commit 39f47f7d authored by j.hoerdt's avatar j.hoerdt
Browse files

merge master into 15-new-auth... to find issues

Squashed commit of the following:

commit aaa9ed97
Merge: 73d70cc3 f1ebf90f
Author: j.hoerdt <j.hoerdt@stud.uni-goettingen.de>
Date:   Fri Apr 16 11:14:04 2021 +0000

    Merge branch '2-complete-pid-registry-registration-implementation' into 'master'

    Resolve "Complete PID registry registration implementation"

    Closes #2

    See merge request !10

commit f1ebf90f
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Thu Apr 15 20:43:29 2021 +0200

    implement Identifier

commit 8f46dd08
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Thu Apr 15 20:24:11 2021 +0200

    implement description

commit ebd0f533
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Thu Apr 15 20:01:10 2021 +0200

    add sht11 description

commit d0458000
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Thu Apr 15 19:58:57 2021 +0200

    add the final manufacturer

commit 921b3bf1
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Thu Apr 15 14:54:32 2021 +0200

    add more manufacturers and rename P1 to PM10 and P2 to PM2.5

commit 73d70cc3
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Wed Apr 14 17:40:38 2021 +0200

    restriction to master not necessary since :master tag is pulled

commit f33c4377
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Wed Apr 14 17:18:13 2021 +0200

    switch to dedicated kaniko repository for ci

commit f2bcd64b
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Tue Apr 13 19:38:37 2021 +0200

    add some BME280 info

commit c657a003
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Tue Apr 13 19:37:09 2021 +0200

    add skeleton instrument type

commit 6e6f89bb
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Tue Apr 13 19:36:45 2021 +0200

    add manufacturers

commit ae005b7e
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Tue Apr 13 19:35:57 2021 +0200

    produce correct json for string only attributes

commit 664e63af
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Tue Apr 13 19:23:55 2021 +0200

    bump neo4j driver and move to maven repository

commit 6076716a
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Tue Apr 13 18:27:08 2021 +0200

    add Model according to spec and fix quotes

commit 389f91e1
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Tue Apr 13 17:54:09 2021 +0200

    implement date according to schema

commit c4a7adc2
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Mon Apr 12 20:05:24 2021 +0200

    implement Owners property validly

commit 2d236977
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Mon Apr 12 19:45:40 2021 +0200

    implement valid LandingPage

commit b545394d
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Mon Apr 12 18:19:24 2021 +0200

    add all sensor_types

commit 5849aa6f
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Tue Mar 16 17:14:19 2021 +0100

    create SensorTypes at the beginning of a run

commit 80dbefb8
Author: neop <j.hoerdt@stud.uni-goettingen.de>
Date:   Tue Mar 16 14:39:54 2021 +0100

    demo cypher query able to produce all SensorType nodes
parent 7c896ce1
......@@ -9,3 +9,4 @@
!*.gradle
!*.pem
!*.properties
!sensor_type_info.json
\ No newline at end of file
*
!*/
# whitelist
!gradle/*
!src/**/*
!.dockerignore
!Dockerfile
!docker-compose.yaml
!.gitattributes
!.gitignore
!.gitlab-ci.yml
!build.gradle
!settings.gradle
!gradlew
!gradlew.bat
!run.sh
!LICENSE
!README.md
!logging.properties
!sensor2graph.properties
!graphdb.gwdg.de.pem
# blacklist:
#.gradle
#build
#.classpath
#.project
#.settings
#.factorypath
#.vscode
#bin
#data
#*.log
#*.html
#already_uploaded_days.txt
#failed_to_upload.txt
*
!*/
# whitelist
!gradle/*
!src/**/*
!.dockerignore
!Dockerfile
!docker-compose.yaml
!.gitattributes
!.gitignore
!.gitlab-ci.yml
!build.gradle
!settings.gradle
!gradlew
!gradlew.bat
!run.sh
!LICENSE
!README.md
!logging.properties
!sensor2graph.properties
!graphdb.gwdg.de.pem
!sensor_type_info.json
!sensor_info/**/*
# blacklist:
#.gradle
#build
#.classpath
#.project
#.settings
#.factorypath
#.vscode
#bin
#data
#*.log
#*.html
#already_uploaded_days.txt
#failed_to_upload.txt
#*.lck
\ No newline at end of file
......@@ -14,11 +14,8 @@ buildjarfile:
buildimage:
only:
refs:
- master
image:
name: $CI_REGISTRY/j.hoerdt/sensor2graph/kaniko
name: $CI_REGISTRY/j.hoerdt/kaniko
entrypoint: [""]
script:
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
......
......@@ -10,7 +10,7 @@ repositories {
dependencies {
testImplementation 'junit:junit:4.12'
implementation 'org.neo4j.driver:neo4j-java-driver:4.2.1'
implementation 'org.neo4j.driver:neo4j-java-driver:4.2.4'
implementation 'com.google.code.gson:gson:2.8.6'
implementation 'com.squareup.okhttp3:okhttp:4.9.1'
}
......
info = [
(
["pressure", "altitude", "pressure_sealevel", "temperature", "humidity"],
"BME280",
),
(["pressure", "altitude", "pressure_sealevel", "temperature"], "BMP180"),
(["pressure", "altitude", "pressure_sealevel", "temperature"], "BMP280"),
(["temperature", "humidity"], "DHT22"),
(["temperature"], "DS18B20"),
(["P1", "P2"], "HPM"),
(["temperature", "humidity"], "HTU21D"),
(
["noise_LAeq", "noise_LA_min", "noise_LA_max", "noise_LA01", "noise_LA95"],
"Laerm",
),
(["P1", "P2", "P0"], "PMS1003"),
(["P1", "P2", "P0"], "PMS3003"),
(["P1", "P2", "P0"], "PMS5003"),
(["P1", "P2", "P0"], "PMS6003"),
(["P1", "P2", "P0"], "PMS7003"),
(["P1", "durP1", "ratioP1", "P2", "durP2", "ratioP2"], "PPD42NS"),
(
["counts_per_minute", "hv_pulses", "tube", "counts", "sample_time_ms"],
"Radiation_SBM-19",
),
(
["counts_per_minute", "hv_pulses", "tube", "counts", "sample_time_ms"],
"Radiation_SBM-20",
),
(
["counts_per_minute", "hv_pulses", "tube", "counts", "sample_time_ms"],
"Radiation_Si22G",
),
(["P1", "durP1", "ratioP1", "P2", "durP2", "ratioP2"], "SDS011"),
(["temperature", "humidity"], "SHT11"),
(["temperature", "humidity"], "SHT30"),
(["temperature", "humidity"], "SHT31"),
(["temperature", "humidity"], "SHT35"),
(["temperature", "humidity"], "SHT85"),
(["P1", "P4", "P2", "P0", "N10", "N4", "N25", "N1", "N05", "TS"], "SPS30"),
]
import json
print(json.dumps({ "sensor_type_info": [{
"measured_variables": variables,
"props": {
"manufacturers": "asdf",
"description": "asdf",
"instrument_type": "asdf",
"sensor_type": typ
}
} for variables, typ in info]}))
{
"sensor_type_info": [
{
"measured_variables": [
"pressure",
"altitude",
"pressure_sealevel",
"temperature",
"humidity"
],
"props": {
"manufacturer": "Bosch Sensortec GmbH",
"description": "The BME280 is a humidity sensor especially developed for mobile applications and wearables where size and low power consumption are key design parameters. The unit combines high linearity and high accuracy sensors and is perfectly feasible for low current consumption, long-term stability and high EMC robustness. The humidity sensor offers an extremely fast response time and therefore supports performance requirements for emerging applications such as context awareness, and high accuracy over a wide temperature range.",
"sensor_type": "BME280"
}
},
{
"measured_variables": [
"pressure",
"altitude",
"pressure_sealevel",
"temperature"
],
"props": {
"manufacturer": "Bosch Sensortec GmbH",
"sensor_type": "BMP180"
}
},
{
"measured_variables": [
"pressure",
"altitude",
"pressure_sealevel",
"temperature"
],
"props": {
"manufacturer": "Bosch Sensortec GmbH",
"description": "The BMP280 is an absolute barometric pressure sensor, which is especially feasible for mobile applications. Its small dimensions and its low power consumption allow for the implementation in battery-powered devices such as mobile phones, GPS modules or watches. The BMP280 is based on Bosch’s proven piezo-resistive pressure sensor technology featuring high accuracy and linearity as well as long-term stability and high EMC robustness. Numerous device operation options guarantee for highest flexibility. The device is optimized in terms of power consumption, resolution and filter performance.",
"sensor_type": "BMP280"
}
},
{
"measured_variables": [
"temperature",
"humidity"
],
"props": {
"manufacturer": "Aosong Electronics Co.,Ltd",
"sensor_type": "DHT22"
}
},
{
"measured_variables": [
"temperature"
],
"props": {
"manufacturer": "Maxim Integrated Products, Inc.",
"sensor_type": "DS18B20"
}
},
{
"measured_variables": [
"PM10",
"PM2.5"
],
"props": {
"manufacturer": "Honeywell International Inc.",
"sensor_type": "HPM"
}
},
{
"measured_variables": [
"temperature",
"humidity"
],
"props": {
"manufacturer": "TE Connectivity",
"sensor_type": "HTU21D"
}
},
{
"measured_variables": [
"noise_LAeq",
"noise_LA_min",
"noise_LA_max",
"noise_LA01",
"noise_LA95"
],
"props": {
"sensor_type": "Laerm"
}
},
{
"measured_variables": [
"PM10",
"PM2.5",
"P0"
],
"props": {
"manufacturer": "PlanTower",
"sensor_type": "PMS1003"
}
},
{
"measured_variables": [
"PM10",
"PM2.5",
"P0"
],
"props": {
"manufacturer": "PlanTower",
"sensor_type": "PMS3003"
}
},
{
"measured_variables": [
"PM10",
"PM2.5",
"P0"
],
"props": {
"manufacturer": "PlanTower",
"sensor_type": "PMS5003"
}
},
{
"measured_variables": [
"PM10",
"PM2.5",
"P0"
],
"props": {
"manufacturer": "PlanTower",
"sensor_type": "PMS6003"
}
},
{
"measured_variables": [
"PM10",
"PM2.5",
"P0"
],
"props": {
"manufacturer": "PlanTower",
"sensor_type": "PMS7003"
}
},
{
"measured_variables": [
"PM10",
"durP1",
"ratioP1",
"PM2.5",
"durP2",
"ratioP2"
],
"props": {
"manufacturer": "Shinyei",
"sensor_type": "PPD42NS"
}
},
{
"measured_variables": [
"counts_per_minute",
"hv_pulses",
"tube",
"counts",
"sample_time_ms"
],
"props": {
"manufacturer": "Russian Federation",
"sensor_type": "Radiation_SBM-19"
}
},
{
"measured_variables": [
"counts_per_minute",
"hv_pulses",
"tube",
"counts",
"sample_time_ms"
],
"props": {
"manufacturer": "Russian Federation",
"sensor_type": "Radiation_SBM-20"
}
},
{
"measured_variables": [
"counts_per_minute",
"hv_pulses",
"tube",
"counts",
"sample_time_ms"
],
"props": {
"manufacturer": "Russian Federation",
"sensor_type": "Radiation_Si22G"
}
},
{
"measured_variables": [
"PM10",
"durP1",
"ratioP1",
"PM2.5",
"durP2",
"ratioP2"
],
"props": {
"manufacturer": "Nova Fitness Co., Ltd. ",
"sensor_type": "SDS011"
}
},
{
"measured_variables": [
"temperature",
"humidity"
],
"props": {
"manufacturer": "Sensirion AG",
"description": "SHT1x (including SHT10, SHT11 and SHT15) is Sensirion’s family of surface mountable relative humidity and temperature sensors. The sensors integrate sensor elements plus signal processing on a tiny foot print and provide a fully calibrated digital output. A unique capacitive sensor element is used for measuring relative humidity while temperature is measured by a band-gap sensor. The applied CMOSens® technology guarantees excellent reliability and long term stability. Both sensors are seamlessly coupled to a 14bit analog to digital converter and a serial interface circuit. This results in superior signal quality, a fast response time and insensitivity to external disturbances (EMC). Each SHT1x is individually calibrated in a precision humidity chamber. The calibration coefficients are programmed into an OTP memory on the chip. These coefficients are used to internally calibrate the signals from the sensors. The 2-wire serial interface and internal voltage regulation allows for easy and fast system integration. The tiny size and low power consumption makes SHT1x the ultimate choice for even the most demanding applications. SHT1x is supplied in a surface-mountable LCC (Leadless Chip Carrier) which is approved for standard reflow soldering processes. The same sensor is also available with pins (SHT7x) or on flex print (SHTA1).",
"sensor_type": "SHT11"
}
},
{
"measured_variables": [
"temperature",
"humidity"
],
"props": {
"manufacturer": "Sensirion AG",
"sensor_type": "SHT30"
}
},
{
"measured_variables": [
"temperature",
"humidity"
],
"props": {
"manufacturer": "Sensirion AG",
"sensor_type": "SHT31"
}
},
{
"measured_variables": [
"temperature",
"humidity"
],
"props": {
"manufacturer": "Sensirion AG",
"sensor_type": "SHT35"
}
},
{
"measured_variables": [
"temperature",
"humidity"
],
"props": {
"manufacturer": "Sensirion AG",
"sensor_type": "SHT85"
}
},
{
"measured_variables": [
"PM10",
"P4",
"PM2.5",
"P0",
"N10",
"N4",
"N25",
"N1",
"N05",
"TS"
],
"props": {
"manufacturer": "Sensirion AG",
"description": "The SPS30 Particulate Matter (PM) sensor is a technological breakthrough in optical PM sensors. Its measurement principle is based on laser scattering and makes use of Sensirion's innovative contaminationresistance technology. This technology, together with high-quality and long-lasting components, enables precise measurements from its first operation and throughout its lifetime of more than ten years. In addition, Sensirion's advanced algorithms provide superior precision for different PM types and higher-resolution particle size binning, opening up new possibilities for the detection of different sorts of environmental dust and other particles. With dimensions of only 41 x 41 x 12 mm3, it is also the perfect solution for applications where size is of paramount importance, such as wall-mounted or compact air quality devices.",
"sensor_type": "SPS30"
}
}
]
}
\ No newline at end of file
......@@ -3,6 +3,9 @@ package sensor2graph;
import org.neo4j.driver.*;
import org.neo4j.driver.Config.*;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import sensor2graph.webdirget.*;
import java.io.*;
......@@ -54,8 +57,8 @@ public class Main {
//.withMaxTransactionRetryTime(3, TimeUnit.SECONDS)
//.withLeakedSessionsLogging()
.withMaxConnectionPoolSize(concurrency)
.withEncryption()
.withTrustStrategy(TrustStrategy.trustCustomCertificateSignedBy(neo4j_server_certificate_file))
// .withEncryption()
// .withTrustStrategy(TrustStrategy.trustCustomCertificateSignedBy(neo4j_server_certificate_file))
.build()
);
}
......@@ -166,7 +169,8 @@ public class Main {
try {
Handle.http_client = Handle.get_client();
test_auth();
run();
pre_run_graphdb_actions();
//run();
} catch (org.neo4j.driver.exceptions.AuthenticationException e) {
Main.logger.log(Level.SEVERE, "Authentication failed, are your credentials correct?");
} catch (Exception e) {
......@@ -246,6 +250,26 @@ public class Main {
}
private static void pre_run_graphdb_actions() throws Exception {
Map<String, Object> parameters = new Gson().fromJson(new FileReader("sensor_type_info.json"), new TypeToken <Map<String, Object>>() {}.getType());
try (var session = Main.driver.session(SessionConfig.forDatabase(Main.database_name))) {
session.writeTransaction(tx -> {
tx.run(
"unwind $sensor_type_info as sensor_type_info\n" +
"merge (n:SensorType {sensor_type: sensor_type_info.props.sensor_type})\n" +
"set n = sensor_type_info.props\n" +
"with n, sensor_type_info\n" +
"unwind sensor_type_info.measured_variables as measured_variable\n" +
"merge (m:MeasuredVariable {variable: measured_variable})\n" +
"merge (n)-[:MEASURES]->(m)\n",
parameters
);
return 42;
});
}
}
private static void test_auth() {
Main.logger.fine("testing authentication");
try (Session session = Main.driver.session(SessionConfig.forDatabase(Main.database_name))) {
......
......@@ -18,7 +18,7 @@ class Sensor {
private Map<String, Object> properties = new HashMap<String, Object>();
public void process(Session session, TransactionConfig transaction_config) throws Exception {
org.neo4j.driver.Record rec = session.writeTransaction(tx -> {
var rec = session.writeTransaction(tx -> {
Result res = tx.run(get_creation_query());
return res.single();
}, transaction_config);
......@@ -58,12 +58,47 @@ class Sensor {
);
add_attribute(body,
"21.T11148/22c62082a4d2d9ae2602",
"[{\"Date\":{\"date\":\"" + rec.get("min(all_measurements.first_msg)").asLocalDate().toString() + "\",\"dateType\":\"first measurement\"}},{\"Date\":{\"date\":\"" + rec.get("max(all_measurements.last_msg)").asLocalDate().toString() + "\",\"dateType\":\"last measurement\"}}]"
"[{\"dateType\":\"Commissioned\", \"date\":\"" + rec.get("min(all_measurements.first_msg)").asLocalDate().toString() + "\"},{\"dateType\":\"last message\", \"date\":\"" + rec.get("max(all_measurements.last_msg)").asLocalDate().toString() + "\"}]"
);
add_attribute(body,
"21.T11148/709a23220f2c3d64d1e1",
rec.get("sensor_type").asString()
'\"' + rec.get("sensor_type").asString() + '\"'
);
add_attribute(body,
"21.T11148/68c2cc7f0ceaa3e499ec",
"{\"Model\":{\"modelName\":\"" + rec.get("sensor_type").asString() + "\"}}"
);
add_attribute(body,
"21.T11148/9a15a4735d4bda329d80",
"\"https://sensor.community\""
);
add_attribute(body,
"21.T11148/4eaec4bc0f1df68ab2a7",
"{\"Owners\":[{\"Owner\":{\"ownerName\":\"SensorCommunity\",\"ownerContact\":\"contact@open-forecast.eu\"}}]}"
);
add_attribute(body,
"21.T11148/1f3e82ddf0697a497432",
"{\"Manufacturers\":[{\"Manufacturer\":{\"manufacturerName\":\""+ rec.get("manufacturer").asString() +"\"}}]}"
);
if (!rec.get("description").isNull()) {
add_attribute(body,
"21.T11148/55f8ebc805e65b5b71dd",
"\""+ rec.get("description") +"\""
);
}
if (!rec.get("s.pid").isNull()) {
add_attribute(body,
"21.T11148/8eb858ee0b12e8e463a5",
"{\"identifier-general-with-type\":{\"identifierValue\":\""+ rec.get("s.pid") +"\", \"identifierType\":\"Handle\"}}"
);
}
// add_attribute(body,
// "21.T11148/f76ad9d0324302fc47dd",
// "\"instrument type string\""
// );
Main.glogger.fine("sent body: " + body);
......@@ -133,9 +168,9 @@ class Sensor {
" measurement.first_msg = case when date_ > measurement.first_msg then measurement.first_msg else date_ end," +
" measurement.last_msg = case when date_ < measurement.last_msg then measurement.last_msg else date_ end " +
"with s, type.sensor_type as sensor_type\n" +
"with s, type.sensor_type as sensor_type, type.manufacturer as manufacturer, type.description as description\n" +
"match (s)--(all_measurements:MeasurementLocation)" +
"return s.pid, id(s), sensor_type, min(all_measurements.first_msg), max(all_measurements.last_msg)"
"return s.pid, id(s), sensor_type, manufacturer, description min(all_measurements.first_msg), max(all_measurements.last_msg)"
).withParameters(properties);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment