Commit: 39ce1836a9db8561243039b6171b232c02071bb6 - myprojects-hg-reborn (hg)

Commit MetaInfo

• Revisión: 39ce1836a9db8561243039b6171b232c02071bb6 (tree)
• Tiempo: 2024-09-18 02:45:30
• Autor: Lorenzo Isella <lorenzo.isella@gmai...>
• Commiter: Lorenzo Isella

Log Message

I improved the script and handle better the file overwriting.

Cambiar Resumen

• modified: R-codes/create_tam_parquet.R

Diferencia incremental

diff -r b1a2d9f03553 -r 39ce1836a9db R-codes/create_tam_parquet.R
--- a/R-codes/create_tam_parquet.R	Tue Sep 17 14:40:35 2024 +0200
+++ b/R-codes/create_tam_parquet.R	Tue Sep 17 19:45:30 2024 +0200
@@ -20,7 +20,7 @@
 if (start_from_tsv==1) {
 
 
-data <- open_dataset("export.tsv",
+data <- open_dataset("../input/csv_files/export.tsv",
   format = "tsv",
   skip_rows = 1,
   newlines_in_values=T,
@@ -61,12 +61,12 @@
 print("OK HERE")
     
 
-
+remove_files_with_pattern("../input/parquet_data_input/*parquet")
 
     write_dataset(
   data,
   format = "parquet",
-  path = "./parquet_data_input/",
+  path = "../input/parquet_data_input/",
   basename_template="tam_raw-{i}.parquet" ,
   max_rows_per_file = 5e5
 )
@@ -80,13 +80,13 @@
 print("OK HERE2")
 
 
-df_tam_ini <- open_dataset("./parquet_data_input/")
+df_tam_ini <- open_dataset("../input/parquet_data_input/")
 
 
 
 
 
-covid <- open_dataset("SA-Covid19.csv", format="csv") |>
+covid <- open_dataset("../input/csv_files/SA-Covid19.csv", format="csv") |>
     rename("case_reference"="Case Reference") |>
     select(case_reference) |>
     mutate(is_covid_case="Yes") |>
@@ -223,10 +223,12 @@
 }
 
 
+remove_files_with_pattern("../input/parquet-files/tam-original/*parquet")
+
     write_dataset(
   df_new,
   format = "parquet",
-  path = "./data_output/",
+  path = "../input/parquet-files/tam-original",
   basename_template="tam_database_cleaned-{i}.parquet" ,
   max_rows_per_file = 5e5
 )
@@ -236,14 +238,14 @@
 
 
 
-cases_wrong <- df_new |>
-    filter(is.na(granted_aid_absolute_eur),
-           is.na(nominal_aid_absolute_eur),
-           granted_range_eur=="0 - "
-           ) |>
-    collect()
+## cases_wrong <- df_new |>
+##     filter(is.na(granted_aid_absolute_eur),
+##            is.na(nominal_aid_absolute_eur),
+##            granted_range_eur=="0 - "
+##            ) |>
+##     collect()
 
-save_excel(cases_wrong, "tam_errors.xlsx")
+## save_excel(cases_wrong, "tam_errors.xlsx")