Commit: 5f02e8dbea0d074338c3042c14ad48d3e5a4a71a - myprojects-hg-reborn (hg)

Commit MetaInfo

• Revisión: 5f02e8dbea0d074338c3042c14ad48d3e5a4a71a (tree)
• Tiempo: 2024-03-15 21:34:46
• Autor: Lorenzo Isella <lorenzo.isella@gmai...>
• Commiter: Lorenzo Isella

Log Message

I cleaned up the code and I also generate a parquet file.

Cambiar Resumen

• modified: R-codes/process_spanish_new_data_sequentially.R

Diferencia incremental

diff -r 93416964f8e9 -r 5f02e8dbea0d R-codes/process_spanish_new_data_sequentially.R
--- a/R-codes/process_spanish_new_data_sequentially.R	Fri Mar 15 12:38:28 2024 +0100
+++ b/R-codes/process_spanish_new_data_sequentially.R	Fri Mar 15 13:34:46 2024 +0100
@@ -8,21 +8,11 @@
 
 source("/home/lorenzo/myprojects-hg/R-codes/stat_lib.R")
 
-## data <- open_dataset("./", format="csv")
 
-## write_dataset(
-##   data,
-##   format = "parquet",
-##   path = "~/initial",
-##   max_rows_per_file = 1e7
-## )
+read_seq <- 0:250
 
 
-
-read_seq <- 1:249
-
-
-df_map <- read_csv("spanish-mapping.csv") |>
+df_map <- read_csv("../spanish-mapping.csv") |>
     mutate(across(where(is.character),~remove_trailing_spaces(.x)))
 
 
@@ -46,12 +36,12 @@
 
 
 
-mm <- read_excel("../ES_matching.xlsx") %>%
-    clean_names() %>%
-    select(-translate) %>%
+mm <- read_excel("../ES_matching.xlsx") |> 
+    clean_names()  |> 
+    select(-translate) |> 
     mutate(original=make_clean_names(original),
-           match=make_clean_names(match)) %>%
-    select(match, original) %>% 
+           match=make_clean_names(match)) |> 
+    select(match, original) |> 
     pattern_to_pattern("na", "na_1")
 
 
@@ -111,13 +101,14 @@
 
 
 
-df_tot2 <- df_tot %>% rename(!!! mm2) %>%
-    select(-starts_with("na_")) %>%
-    mutate(beneficiary_country="Spain") %>%
-    mutate(across(contains("date"), ~as_date(.x))) %>%  
-    mutate(year=ymd(aid_award_granted_date)) %>% 
-    mutate(year=year(year)) %>%
-    mutate(nominal_aid_absolute_eur=str_replace_all(nominal_aid_absolute_eur,",",""),
+    df_tot2 <- df_tot |>
+        rename(!!! mm2) |> 
+        select(-starts_with("na_"))  |> 
+        mutate(beneficiary_country="Spain") |> 
+        mutate(across(contains("date"), ~as_date(.x))) |> 
+        mutate(year=ymd(aid_award_granted_date)) |> 
+        mutate(year=year(year)) |> 
+        mutate(nominal_aid_absolute_eur=str_replace_all(nominal_aid_absolute_eur,",",""),
            granted_aid_absolute_eur=str_replace_all(granted_aid_absolute_eur,",","")) |>
     mutate(across(contains("aid_absolute"),  ~as.numeric(.x))) |>
     mutate(across(contains("date"), ~ymd(.x)))|>
@@ -147,15 +138,48 @@
 write_csv(df_tot2, fname.out)
     
 }
-    
-## saveRDS(df_tot2, "tam_ES_matched.RDS")
-
-## write_csv(df_tot2, "spanish_tam_cleaned.csv.gz")
 
 
-## save_excel(df_tot2, "spanish_tam.xlsx")
 
-## write_parquet(df_tot2, "spanish_tam.parquet")
+system("csvstack processed_data*csv > final_spain.csv")
+
+data <- open_dataset("final_spain.csv",
+  format = "csv",
+  skip_rows = 1, 
+  schema = schema(
+      aid_award_ga_original = string(),
+      aid_award_reference = string(),
+      aid_award_objective = string(),
+      aid_award_instrument = string(),
+      beneficiary_type = string(),
+      aid_award_granted_date = string(),
+      aid_award_published_date= string(),
+      beneficiary_name= string(),
+      nominal_aid_absolute_eur= double(),
+      granted_aid_absolute_eur = double(),
+      beneficiary_region = string(),
+      beneficiary_sector = string(),
+      case_reference = string(),
+      beneficiary_country = string(),
+      year= double()    ,
+      nominal_value_extended_eur = double(),
+      granted_value_extended_eur = double(),
+      is_covid_case = string(),
+      national_identification= string()
+  )
+)
+
+## write_dataset(
+##   data,
+##   format = "parquet",
+##   path = ".",
+##   max_rows_per_file = 1e7
+## )
+
+write_parquet(
+    data, "final_spain.parquet")
+
+