diff --git a/.dockerignore b/.dockerignore index d23268657334dfd8dd5c5e8710fdac7bc3844d59..b60a27345e01c0a5e2c858d406d91e39a13a116b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,4 @@ !*.gradle !*.pem !*.properties +!sensor_type_info.json \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8c6ee34125a445ab0b58620a078434ed0a6b555e..be82de5f9bb7959bdc5d34d8354fde1920d67225 100644 --- a/.gitignore +++ b/.gitignore @@ -1,43 +1,45 @@ -* -!*/ - -# whitelist -!gradle/* -!src/**/* -!.dockerignore -!Dockerfile -!docker-compose.yaml -!.gitattributes -!.gitignore -!.gitlab-ci.yml -!build.gradle -!settings.gradle -!gradlew -!gradlew.bat -!run.sh -!LICENSE -!README.md -!logging.properties -!sensor2graph.properties -!graphdb.gwdg.de.pem - - - - - -# blacklist: - -#.gradle -#build -#.classpath -#.project -#.settings -#.factorypath -#.vscode -#bin -#data -#*.log -#*.html -#already_uploaded_days.txt -#failed_to_upload.txt +* +!*/ + +# whitelist +!gradle/* +!src/**/* +!.dockerignore +!Dockerfile +!docker-compose.yaml +!.gitattributes +!.gitignore +!.gitlab-ci.yml +!build.gradle +!settings.gradle +!gradlew +!gradlew.bat +!run.sh +!LICENSE +!README.md +!logging.properties +!sensor2graph.properties +!graphdb.gwdg.de.pem +!sensor_type_info.json +!sensor_info/**/* + + + + + +# blacklist: + +#.gradle +#build +#.classpath +#.project +#.settings +#.factorypath +#.vscode +#bin +#data +#*.log +#*.html +#already_uploaded_days.txt +#failed_to_upload.txt #*.lck \ No newline at end of file diff --git a/sensor_info/sensor_info_skeleton_generator.py b/sensor_info/sensor_info_skeleton_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..57e57fc63fb23960fdd9044ef6f7acd8a3e5801e --- /dev/null +++ b/sensor_info/sensor_info_skeleton_generator.py @@ -0,0 +1,55 @@ +info = [ + ( + ["pressure", "altitude", "pressure_sealevel", "temperature", "humidity"], + "BME280", + ), + (["pressure", "altitude", "pressure_sealevel", "temperature"], "BMP180"), + (["pressure", "altitude", "pressure_sealevel", "temperature"], "BMP280"), + (["temperature", "humidity"], "DHT22"), + (["temperature"], "DS18B20"), + (["P1", "P2"], "HPM"), + (["temperature", "humidity"], "HTU21D"), + ( + ["noise_LAeq", "noise_LA_min", "noise_LA_max", "noise_LA01", "noise_LA95"], + "Laerm", + ), + (["P1", "P2", "P0"], "PMS1003"), + (["P1", "P2", "P0"], "PMS3003"), + (["P1", "P2", "P0"], "PMS5003"), + (["P1", "P2", "P0"], "PMS6003"), + (["P1", "P2", "P0"], "PMS7003"), + (["P1", "durP1", "ratioP1", "P2", "durP2", "ratioP2"], "PPD42NS"), + ( + ["counts_per_minute", "hv_pulses", "tube", "counts", "sample_time_ms"], + "Radiation_SBM-19", + ), + ( + ["counts_per_minute", "hv_pulses", "tube", "counts", "sample_time_ms"], + "Radiation_SBM-20", + ), + ( + ["counts_per_minute", "hv_pulses", "tube", "counts", "sample_time_ms"], + "Radiation_Si22G", + ), + (["P1", "durP1", "ratioP1", "P2", "durP2", "ratioP2"], "SDS011"), + (["temperature", "humidity"], "SHT11"), + (["temperature", "humidity"], "SHT30"), + (["temperature", "humidity"], "SHT31"), + (["temperature", "humidity"], "SHT35"), + (["temperature", "humidity"], "SHT85"), + (["P1", "P4", "P2", "P0", "N10", "N4", "N25", "N1", "N05", "TS"], "SPS30"), +] + +import json + +print(json.dumps({ "sensor_type_info": [{ + "measured_variables": variables, + "props": { + "manufacturers": "asdf", + "description": "asdf", + "instrument_type": "asdf", + "sensor_type": typ + } +} for variables, typ in info]})) + + diff --git a/sensor_type_info.json b/sensor_type_info.json new file mode 100644 index 0000000000000000000000000000000000000000..e9b5ead0dc250f9b5ea89974d9317992a6620342 --- /dev/null +++ b/sensor_type_info.json @@ -0,0 +1,286 @@ +{ + "sensor_type_info": [ + { + "measured_variables": [ + "pressure", + "altitude", + "pressure_sealevel", + "temperature", + "humidity" + ], + "props": { + "manufacturer": "Bosch Sensortec GmbH", + "description": "The BME280 is a humidity sensor especially developed for mobile applications and wearables where size and low power consumption are key design parameters. The unit combines high linearity and high accuracy sensors and is perfectly feasible for low current consumption, long-term stability and high EMC robustness. The humidity sensor offers an extremely fast response time and therefore supports performance requirements for emerging applications such as context awareness, and high accuracy over a wide temperature range.", + "sensor_type": "BME280" + } + }, + { + "measured_variables": [ + "pressure", + "altitude", + "pressure_sealevel", + "temperature" + ], + "props": { + "manufacturer": "Bosch Sensortec GmbH", + "sensor_type": "BMP180" + } + }, + { + "measured_variables": [ + "pressure", + "altitude", + "pressure_sealevel", + "temperature" + ], + "props": { + "manufacturer": "Bosch Sensortec GmbH", + "description": "The BMP280 is an absolute barometric pressure sensor, which is especially feasible for mobile applications. Its small dimensions and its low power consumption allow for the implementation in battery-powered devices such as mobile phones, GPS modules or watches. The BMP280 is based on Bosch’s proven piezo-resistive pressure sensor technology featuring high accuracy and linearity as well as long-term stability and high EMC robustness. Numerous device operation options guarantee for highest flexibility. The device is optimized in terms of power consumption, resolution and filter performance.", + "sensor_type": "BMP280" + } + }, + { + "measured_variables": [ + "temperature", + "humidity" + ], + "props": { + "manufacturer": "Aosong Electronics Co.,Ltd", + "sensor_type": "DHT22" + } + }, + { + "measured_variables": [ + "temperature" + ], + "props": { + "manufacturer": "Maxim Integrated Products, Inc.", + "sensor_type": "DS18B20" + } + }, + { + "measured_variables": [ + "PM10", + "PM2.5" + ], + "props": { + "manufacturer": "Honeywell International Inc.", + "sensor_type": "HPM" + } + }, + { + "measured_variables": [ + "temperature", + "humidity" + ], + "props": { + "manufacturer": "TE Connectivity", + "sensor_type": "HTU21D" + } + }, + { + "measured_variables": [ + "noise_LAeq", + "noise_LA_min", + "noise_LA_max", + "noise_LA01", + "noise_LA95" + ], + "props": { + "sensor_type": "Laerm" + } + }, + { + "measured_variables": [ + "PM10", + "PM2.5", + "P0" + ], + "props": { + "manufacturer": "PlanTower", + "sensor_type": "PMS1003" + } + }, + { + "measured_variables": [ + "PM10", + "PM2.5", + "P0" + ], + "props": { + "manufacturer": "PlanTower", + "sensor_type": "PMS3003" + } + }, + { + "measured_variables": [ + "PM10", + "PM2.5", + "P0" + ], + "props": { + "manufacturer": "PlanTower", + "sensor_type": "PMS5003" + } + }, + { + "measured_variables": [ + "PM10", + "PM2.5", + "P0" + ], + "props": { + "manufacturer": "PlanTower", + "sensor_type": "PMS6003" + } + }, + { + "measured_variables": [ + "PM10", + "PM2.5", + "P0" + ], + "props": { + "manufacturer": "PlanTower", + "sensor_type": "PMS7003" + } + }, + { + "measured_variables": [ + "PM10", + "durP1", + "ratioP1", + "PM2.5", + "durP2", + "ratioP2" + ], + "props": { + "manufacturer": "Shinyei", + "sensor_type": "PPD42NS" + } + }, + { + "measured_variables": [ + "counts_per_minute", + "hv_pulses", + "tube", + "counts", + "sample_time_ms" + ], + "props": { + "manufacturer": "Russian Federation", + "sensor_type": "Radiation_SBM-19" + } + }, + { + "measured_variables": [ + "counts_per_minute", + "hv_pulses", + "tube", + "counts", + "sample_time_ms" + ], + "props": { + "manufacturer": "Russian Federation", + "sensor_type": "Radiation_SBM-20" + } + }, + { + "measured_variables": [ + "counts_per_minute", + "hv_pulses", + "tube", + "counts", + "sample_time_ms" + ], + "props": { + "manufacturer": "Russian Federation", + "sensor_type": "Radiation_Si22G" + } + }, + { + "measured_variables": [ + "PM10", + "durP1", + "ratioP1", + "PM2.5", + "durP2", + "ratioP2" + ], + "props": { + "manufacturer": "Nova Fitness Co., Ltd. ", + "sensor_type": "SDS011" + } + }, + { + "measured_variables": [ + "temperature", + "humidity" + ], + "props": { + "manufacturer": "Sensirion AG", + "description": "SHT1x (including SHT10, SHT11 and SHT15) is Sensirion’s family of surface mountable relative humidity and temperature sensors. The sensors integrate sensor elements plus signal processing on a tiny foot print and provide a fully calibrated digital output. A unique capacitive sensor element is used for measuring relative humidity while temperature is measured by a band-gap sensor. The applied CMOSens® technology guarantees excellent reliability and long term stability. Both sensors are seamlessly coupled to a 14bit analog to digital converter and a serial interface circuit. This results in superior signal quality, a fast response time and insensitivity to external disturbances (EMC). Each SHT1x is individually calibrated in a precision humidity chamber. The calibration coefficients are programmed into an OTP memory on the chip. These coefficients are used to internally calibrate the signals from the sensors. The 2-wire serial interface and internal voltage regulation allows for easy and fast system integration. The tiny size and low power consumption makes SHT1x the ultimate choice for even the most demanding applications. SHT1x is supplied in a surface-mountable LCC (Leadless Chip Carrier) which is approved for standard reflow soldering processes. The same sensor is also available with pins (SHT7x) or on flex print (SHTA1).", + "sensor_type": "SHT11" + } + }, + { + "measured_variables": [ + "temperature", + "humidity" + ], + "props": { + "manufacturer": "Sensirion AG", + "sensor_type": "SHT30" + } + }, + { + "measured_variables": [ + "temperature", + "humidity" + ], + "props": { + "manufacturer": "Sensirion AG", + "sensor_type": "SHT31" + } + }, + { + "measured_variables": [ + "temperature", + "humidity" + ], + "props": { + "manufacturer": "Sensirion AG", + "sensor_type": "SHT35" + } + }, + { + "measured_variables": [ + "temperature", + "humidity" + ], + "props": { + "manufacturer": "Sensirion AG", + "sensor_type": "SHT85" + } + }, + { + "measured_variables": [ + "PM10", + "P4", + "PM2.5", + "P0", + "N10", + "N4", + "N25", + "N1", + "N05", + "TS" + ], + "props": { + "manufacturer": "Sensirion AG", + "description": "The SPS30 Particulate Matter (PM) sensor is a technological breakthrough in optical PM sensors. Its measurement principle is based on laser scattering and makes use of Sensirion's innovative contaminationresistance technology. This technology, together with high-quality and long-lasting components, enables precise measurements from its first operation and throughout its lifetime of more than ten years. In addition, Sensirion's advanced algorithms provide superior precision for different PM types and higher-resolution particle size binning, opening up new possibilities for the detection of different sorts of environmental dust and other particles. With dimensions of only 41 x 41 x 12 mm3, it is also the perfect solution for applications where size is of paramount importance, such as wall-mounted or compact air quality devices.", + "sensor_type": "SPS30" + } + } + ] +} \ No newline at end of file diff --git a/src/main/java/sensor2graph/Main.java b/src/main/java/sensor2graph/Main.java index ea683299639c232a4457e05715fa55b59d71e212..119aa8f8da70e2cd1ff6f6c348939162a435dd0e 100644 --- a/src/main/java/sensor2graph/Main.java +++ b/src/main/java/sensor2graph/Main.java @@ -3,6 +3,9 @@ package sensor2graph; import org.neo4j.driver.*; import org.neo4j.driver.Config.*; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + import sensor2graph.webdirget.*; import java.io.*; @@ -53,8 +56,8 @@ public class Main { //.withMaxTransactionRetryTime(3, TimeUnit.SECONDS) //.withLeakedSessionsLogging() .withMaxConnectionPoolSize(concurrency) - .withEncryption() - .withTrustStrategy(TrustStrategy.trustCustomCertificateSignedBy(neo4j_server_certificate_file)) + // .withEncryption() + // .withTrustStrategy(TrustStrategy.trustCustomCertificateSignedBy(neo4j_server_certificate_file)) .build() ); } @@ -162,7 +165,8 @@ public class Main { try { test_auth(); - run(); + pre_run_graphdb_actions(); + //run(); } catch (org.neo4j.driver.exceptions.AuthenticationException e) { Main.logger.log(Level.SEVERE, "Authentication failed, are your credentials correct?"); } catch (Exception e) { @@ -242,6 +246,26 @@ public class Main { } + private static void pre_run_graphdb_actions() throws Exception { + Map parameters = new Gson().fromJson(new FileReader("sensor_type_info.json"), new TypeToken >() {}.getType()); + try (var session = Main.driver.session(SessionConfig.forDatabase(Main.database_name))) { + session.writeTransaction(tx -> { + tx.run( + "unwind $sensor_type_info as sensor_type_info\n" + + "merge (n:SensorType {sensor_type: sensor_type_info.props.sensor_type})\n" + + "set n = sensor_type_info.props\n" + + "with n, sensor_type_info\n" + + "unwind sensor_type_info.measured_variables as measured_variable\n" + + "merge (m:MeasuredVariable {variable: measured_variable})\n" + + "merge (n)-[:MEASURES]->(m)\n", + parameters + ); + return 42; + }); + } + } + + private static void test_auth() { Main.logger.fine("testing authentication"); try (Session session = Main.driver.session(SessionConfig.forDatabase(Main.database_name))) { diff --git a/src/main/java/sensor2graph/Sensor.java b/src/main/java/sensor2graph/Sensor.java index 27c4a2a8701db82318c77c83673007a8f84659ed..6f5cfd855a4fe59dff124e5fdc05bbb0ae8fb42c 100644 --- a/src/main/java/sensor2graph/Sensor.java +++ b/src/main/java/sensor2graph/Sensor.java @@ -28,7 +28,7 @@ class Sensor { private Map properties = new HashMap(); public void process(Session session, TransactionConfig transaction_config) throws Exception { - org.neo4j.driver.Record rec = session.writeTransaction(tx -> { + var rec = session.writeTransaction(tx -> { Result res = tx.run(get_creation_query()); return res.single(); }, transaction_config); @@ -68,12 +68,47 @@ class Sensor { ); add_attribute(body, "21.T11148/22c62082a4d2d9ae2602", - "[{\"Date\":{\"date\":\"" + rec.get("min(all_measurements.first_msg)").asLocalDate().toString() + "\",\"dateType\":\"first measurement\"}},{\"Date\":{\"date\":\"" + rec.get("max(all_measurements.last_msg)").asLocalDate().toString() + "\",\"dateType\":\"last measurement\"}}]" + "[{\"dateType\":\"Commissioned\", \"date\":\"" + rec.get("min(all_measurements.first_msg)").asLocalDate().toString() + "\"},{\"dateType\":\"last message\", \"date\":\"" + rec.get("max(all_measurements.last_msg)").asLocalDate().toString() + "\"}]" ); add_attribute(body, "21.T11148/709a23220f2c3d64d1e1", - rec.get("sensor_type").asString() + '\"' + rec.get("sensor_type").asString() + '\"' ); + add_attribute(body, + "21.T11148/68c2cc7f0ceaa3e499ec", + "{\"Model\":{\"modelName\":\"" + rec.get("sensor_type").asString() + "\"}}" + ); + add_attribute(body, + "21.T11148/9a15a4735d4bda329d80", + "\"https://sensor.community\"" + ); + add_attribute(body, + "21.T11148/4eaec4bc0f1df68ab2a7", + "{\"Owners\":[{\"Owner\":{\"ownerName\":\"SensorCommunity\",\"ownerContact\":\"contact@open-forecast.eu\"}}]}" + ); + add_attribute(body, + "21.T11148/1f3e82ddf0697a497432", + "{\"Manufacturers\":[{\"Manufacturer\":{\"manufacturerName\":\""+ rec.get("manufacturer").asString() +"\"}}]}" + ); + if (!rec.get("description").isNull()) { + add_attribute(body, + "21.T11148/55f8ebc805e65b5b71dd", + "\""+ rec.get("description") +"\"" + ); + } + if (!rec.get("s.pid").isNull()) { + add_attribute(body, + "21.T11148/8eb858ee0b12e8e463a5", + "{\"identifier-general-with-type\":{\"identifierValue\":\""+ rec.get("s.pid") +"\", \"identifierType\":\"Handle\"}}" + ); + } + // add_attribute(body, + // "21.T11148/f76ad9d0324302fc47dd", + // "\"instrument type string\"" + // ); + + + Main.glogger.fine("sent body: " + body); HttpRequest request = HttpRequest.newBuilder(URI.create(Main.pid_registry_uri + target_uri)) @@ -137,9 +172,9 @@ class Sensor { " measurement.first_msg = case when date_ > measurement.first_msg then measurement.first_msg else date_ end," + " measurement.last_msg = case when date_ < measurement.last_msg then measurement.last_msg else date_ end " + - "with s, type.sensor_type as sensor_type\n" + + "with s, type.sensor_type as sensor_type, type.manufacturer as manufacturer, type.description as description\n" + "match (s)--(all_measurements:MeasurementLocation)" + - "return s.pid, id(s), sensor_type, min(all_measurements.first_msg), max(all_measurements.last_msg)" + "return s.pid, id(s), sensor_type, manufacturer, description min(all_measurements.first_msg), max(all_measurements.last_msg)" ).withParameters(properties); }