Commit d146ad77 authored by neop's avatar neop
Browse files

fix #13, delete archived folders

parent fbda8de4
......@@ -111,35 +111,53 @@ class DayUploader {
}
*/
public static void upload_daily_folder(String daily_folder_uri) throws IOException {
public static void upload_daily_folder(String daily_folder_url) throws Exception {
try {
upload_from_csv_uris(get_csv_files(daily_folder_uri));
create_daily_archive(new URL(daily_folder_uri));
upload_from_csv_uris(get_csv_files(daily_folder_url));
} catch (Exception e) {
Main.glogger.log(Level.SEVERE, "could not import " + daily_folder_uri + ", not adding to uploaded folders", e);
Main.glogger.log(Level.SEVERE, "could not import " + daily_folder_url + ", not adding to uploaded folders", e);
return;
}
create_daily_archive(new URL(daily_folder_url));
remove_daily_folder(new URL(daily_folder_url));
Main.glogger.fine("adding " + daily_folder_uri + " to list of successful days");
Util.append_to_file(Main.already_uploaded_days_file, daily_folder_uri);
Main.glogger.fine("adding " + daily_folder_url + " to list of successful days");
Util.append_to_file(Main.already_uploaded_days_file, daily_folder_url);
}
public static void create_daily_archive(URL daily_folder_url) throws Exception {
String date = daily_folder_url.getFile().substring(1, 11);
String tarname = date + ".tar.zst";
Main.glogger.info("archiving " + date + " to " + tarname);
Files.createDirectories(new File(Main.csv_directory, date).toPath(), daily_folder_perms);
ProcessBuilder tar_zstd = new ProcessBuilder("tar", "--zstd", "-cf", tarname, date);
tar_zstd.directory(Main.csv_directory);
tar_zstd.redirectErrorStream(true);
var tar_zstd = new ProcessBuilder("tar", "--zstd", "-cf", tarname, date)
.directory(Main.csv_directory)
.redirectErrorStream(true);
long start_time = System.currentTimeMillis();
check_process(tar_zstd.start(), "tar", start_time);
run_process(tar_zstd, "tar");
}
public static void remove_daily_folder(URL daily_folder_url) throws Exception {
String date = daily_folder_url.getFile().substring(1, 11);
Main.glogger.info("deleting archived folder " + date);
//easier than stdlib:
var rm = new ProcessBuilder("rm", "-rf", date)
.directory(Main.csv_directory)
.redirectErrorStream(true);
run_process(rm, "rm");
}
public static void check_process(Process p, String name, long start_time) throws Exception {
if (p.waitFor(Main.tar_timeout_minutes, TimeUnit.MINUTES) == false /*waiting time elapsed*/) {
public static void run_process(ProcessBuilder process_builder, String name) throws Exception {
long start_time = System.currentTimeMillis();
var p = process_builder.start();
if (p.waitFor(Main.process_timeout_minutes, TimeUnit.MINUTES) == false /*waiting time elapsed*/) {
throw new TimeoutException(name + " process took too long");
}
......
......@@ -28,7 +28,7 @@ public class Main {
public static File csv_directory;
public static File geocoding_cache;
public static int concurrency;
public static int tar_timeout_minutes;
public static int process_timeout_minutes;
private static String http_agent;
private static String neo4j_uri;
private static File neo4j_server_certificate_file;
......@@ -203,7 +203,7 @@ public class Main {
prefix_for_new_pids = config.getProperty("prefix_for_new_pids" );
pid_reg_server_cert = config.getProperty("pid_reg_server_cert" );
pid_reg_keystore = config.getProperty("pid_reg_keystore" );
tar_timeout_minutes = Integer.parseInt(config.getProperty("tar_timeout_minutes" ));
process_timeout_minutes = Integer.parseInt(config.getProperty("process_timeout_minutes" ));
}
......@@ -231,7 +231,7 @@ public class Main {
defaults.setProperty("prefix_for_new_pids", "21.11138/" );
defaults.setProperty("pid_reg_server_cert", "pid_reg_server_cert.pem" );
defaults.setProperty("pid_reg_keystore", "pid_reg_keystore.p12" );
defaults.setProperty("tar_timeout_minutes", "60" );
defaults.setProperty("process_timeout_minutes", "60" );
return defaults;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment