Commit 684a7720 authored by j.hoerdt's avatar j.hoerdt
Browse files

refactor small issues in many places with help from sonarlint

parent cd19a0e3
......@@ -10,7 +10,7 @@
!Dockerfile
!docker-compose.yaml
!.gitattributes
!.gitignore
!./.gitignore
!.gitlab-ci.yml
!build.gradle
!settings.gradle
......
File mode changed from 100644 to 100755
package sensor2graph;
import java.io.*;
import java.time.Duration;
import java.util.concurrent.*;
import java.util.logging.*;
import java.util.stream.*;
import java.util.*;
import java.net.*;
import java.nio.file.*;
import java.nio.file.attribute.*;
import com.influxdb.client.domain.WritePrecision;
import com.influxdb.client.write.Point;
import org.neo4j.driver.*;
import org.neo4j.driver.SessionConfig;
import org.neo4j.driver.TransactionConfig;
import sensor2graph.webdirget.DirectoryIndexer;
import sensor2graph.webdirget.*;
import java.io.*;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermissions;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.stream.Collectors;
import java.util.stream.Stream;
class DayUploader {
private DayUploader() {}
private static ForkJoinPool thread_pool = new ForkJoinPool(10);
private static TransactionConfig transaction_config = TransactionConfig.builder().withTimeout(Duration.ofMinutes(1)).build();
private static FileAttribute<?> daily_folder_perms = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-xr-x"));
......@@ -68,12 +74,11 @@ class DayUploader {
//guarantee to evaluate entire stream
results.forEach(res -> {});
//Main.glogger.info("successfully uploaded " + results.count() + " sensors");
Main.glogger.info("finished uploading");
};
}
private static void put_data_into_influxdb(File saved_csv_file, Sensor sensor) throws FileNotFoundException, IOException {
private static void put_data_into_influxdb(File saved_csv_file, Sensor sensor) throws IOException {
var tag_columns = List.of("sensor_id", "sensor_type", "location");
var example_parsing_exception = new Object(){ NumberFormatException e = null; };
......@@ -112,7 +117,7 @@ class DayUploader {
Main.influxdb_write_api.flush();
}
private static Map<String, Integer> get_index_of_field_map(String header_line) throws IOException {
private static Map<String, Integer> get_index_of_field_map(String header_line) {
var header = header_line.split(";", -1);
var index_of_field = new HashMap<String, Integer>();
for (int i = 0; i < header.length; ++i) {
......@@ -127,7 +132,7 @@ class DayUploader {
}
}
private static void download_csv_to_disk(File saved_csv_file, String file_uri) throws MalformedURLException, IOException {
private static void download_csv_to_disk(File saved_csv_file, String file_uri) throws IOException {
Files.createDirectories(saved_csv_file.getParentFile().toPath(), daily_folder_perms);
Main.glogger.info("copying csv to " + saved_csv_file);
try (
......@@ -142,7 +147,7 @@ class DayUploader {
private static Map<String, String> read_one_csv_line(BufferedReader csv_reader) throws IOException {
Iterator<String> header = Arrays.asList(csv_reader.readLine().split(";")).iterator();
Iterator<String> data = Arrays.asList(csv_reader.readLine().split(";")).iterator();
Map<String, String> result = new HashMap<String, String>();
var result = new HashMap<String, String>();
while (header.hasNext() && data.hasNext()) {
result.put(header.next(), data.next());
......@@ -163,12 +168,6 @@ class DayUploader {
private static Stream<String> get_csv_files(String csv_folder_uri) throws Exception {
return get_csv_files(Util.reader_of_uri(csv_folder_uri), csv_folder_uri);
}
/*
private static Stream<String> get_csv_files(String csv_folder_uri, String downloaded_index_html) throws Exception {
return get_csv_files(new BufferedReader(new FileReader(downloaded_index_html)), csv_folder_uri);
}
*/
public static void upload_daily_folder(String daily_folder_url) throws Exception {
try {
......@@ -216,7 +215,7 @@ class DayUploader {
long start_time = System.currentTimeMillis();
var p = process_builder.start();
if (p.waitFor(120, TimeUnit.MINUTES) == false /*waiting time elapsed*/) {
if (!p.waitFor(120, TimeUnit.MINUTES) /*waiting time elapsed*/) {
throw new TimeoutException(name + " process took too long");
}
......
package sensor2graph;
import java.io.*;
import java.time.Duration;
import okhttp3.OkHttpClient;
import javax.net.ssl.*;
import java.security.*;
import java.security.cert.*;
import okhttp3.*;
import java.io.FileInputStream;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateFactory;
import java.time.Duration;
public class Handle {
private Handle() {}
public static OkHttpClient http_client;
private static TrustManager[] get_trust_managers() throws Exception {
......@@ -37,16 +40,15 @@ public class Handle {
return (hostname, sslSession) -> true;
}
public static OkHttpClient get_client() throws Exception, NoSuchAlgorithmException, KeyManagementException {
public static OkHttpClient get_client() throws Exception {
var trust_managers = get_trust_managers();
var trust_self_signed_cert_and_provide_client_cert = SSLContext.getInstance("TLS");
trust_self_signed_cert_and_provide_client_cert.init(get_key_managers(), trust_managers, null);
var client = new OkHttpClient.Builder()
return new OkHttpClient.Builder()
.sslSocketFactory(trust_self_signed_cert_and_provide_client_cert.getSocketFactory(), (X509TrustManager) trust_managers[0])
.hostnameVerifier(allowAllHostNames())
.connectTimeout(Duration.ofSeconds(20))
.build();
return client;
}
}
package sensor2graph;
import com.influxdb.client.*;
import org.neo4j.driver.*;
import okhttp3.OkHttpClient;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import sensor2graph.webdirget.*;
import java.io.*;
import java.nio.file.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.function.*;
import java.util.logging.*;
import java.util.stream.*;
import com.influxdb.client.InfluxDBClient;
import com.influxdb.client.InfluxDBClientFactory;
import com.influxdb.client.InfluxDBClientOptions;
import com.influxdb.client.WriteApi;
import okhttp3.OkHttpClient;
import org.neo4j.driver.*;
import sensor2graph.webdirget.DirectoryIndexer;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.logging.Level;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class Main {
public static Properties config;
......
package sensor2graph;
import sensor2graph.georev.*;
import org.neo4j.driver.*;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import okhttp3.MediaType;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import org.neo4j.driver.Record;
import okhttp3.*;
import com.google.gson.*;
import java.time.*;
import java.util.*;
import org.neo4j.driver.*;
import sensor2graph.georev.Geodata;
import sensor2graph.georev.ReverseGeocoder;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.function.BiFunction;
import java.util.function.Predicate;
import java.util.logging.Level;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.function.*;
import java.util.logging.*;
import java.util.regex.*;
class Sensor {
public Map<String, Object> properties = new HashMap<String, Object>();
......
package sensor2graph;
import java.io.*;
import java.net.*;
import java.nio.file.*;
import java.util.*;
import java.util.stream.*;
import javax.xml.parsers.*;
import org.w3c.dom.*;
import org.xml.sax.*;
import java.time.*;
import java.time.format.*;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.ErrorHandler;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.DateTimeException;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.List;
import java.util.Map;
import java.util.stream.IntStream;
import java.util.stream.Stream;
public class Util {
private static int timeout_millis = 120000;
......
package sensor2graph.georev;
import java.util.*;
import java.util.stream.*;
import org.w3c.dom.Document;
import sensor2graph.Util;
import org.w3c.dom.*;
import sensor2graph.*;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class Geodata {
public String country_code, country, state, city, town, village;
......
package sensor2graph.georev;
import java.io.*;
import java.net.*;
import java.util.concurrent.*;
import java.util.logging.*;
import java.util.stream.*;
import javax.xml.parsers.*;
import org.w3c.dom.*;
import sensor2graph.*;
import org.w3c.dom.Document;
import sensor2graph.Main;
import sensor2graph.Util;
import javax.xml.parsers.DocumentBuilder;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URLConnection;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.logging.Level;
import java.util.stream.Stream;
public class ReverseGeocoder {
private static ReverseGeocoder inst;
......
package sensor2graph.webdirget;
import java.io.*;
import java.net.*;
import java.util.*;
import java.util.function.*;
import java.util.regex.*;
import java.util.stream.*;
import java.io.BufferedReader;
import java.io.IOException;
import java.util.List;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
/**
* This class offers functions to search html pages for links
*/
public class DirectoryIndexer {
private static Stream<String> get_relative_subpaths_of_html(BufferedReader html) throws MalformedURLException, IOException {
private DirectoryIndexer() {}
private static Stream<String> get_relative_subpaths_of_html(BufferedReader html) throws IOException {
List<String> html_index_lines;
try (BufferedReader reader = html) {
html_index_lines = reader.lines().collect(Collectors.toList());
html_index_lines = reader.lines().toList();
}
return get_relative_subpaths_of_html(html_index_lines.stream());
}
......@@ -31,24 +34,18 @@ public class DirectoryIndexer {
});
}
private static Stream<String> get_subpaths_of_html(BufferedReader html, String page_uri) throws MalformedURLException, IOException {
private static Stream<String> get_subpaths_of_html(BufferedReader html, String page_uri) throws IOException {
return get_relative_subpaths_of_html(html).map(path -> page_uri + path);
}
private static Predicate<String> is_csv_file = Pattern.compile(".+\\.csv\\z").asMatchPredicate();
private static Predicate<String> is_daily_folder = Pattern.compile(".+\\d{4}-\\d{2}-\\d{2}/\\z").asMatchPredicate();
public static Stream<String> get_csv_files_in_html(BufferedReader html, String page_uri) throws MalformedURLException, IOException {
public static Stream<String> get_csv_files_in_html(BufferedReader html, String page_uri) throws IOException {
return get_subpaths_of_html(html, page_uri).filter(is_csv_file);
}
public static Stream<String> get_daily_folders_in_html(BufferedReader html, String page_uri) throws MalformedURLException, IOException {
public static Stream<String> get_daily_folders_in_html(BufferedReader html, String page_uri) throws IOException {
return get_subpaths_of_html(html, page_uri).filter(is_daily_folder);
}
/*
public static void main(String[] args) throws MalformedURLException, IOException {
DirectoryIndexer.get_csv_files_recursively_from_html(new BufferedReader(new FileReader("../Index of _.html")), "http://archive.sensor.community/").forEach(System.out::println);
}
*/
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment