safegraphinc / safegraphr Goto Github PK

pitt.data <- read_shop(
  filename = "safe-graph-data.zip",
  keeplist = c("patterns", "home_panel_summary.csv"),
  by = "placekey",
  expand_int = "visitors_by_day",
  name = "visits",
  start_date = lubridate::ymd("2018-01-01"))

Error in read_many_patterns(filelist = patfiles, dir = exdir, recursive = FALSE,  : 
  Number of files (0) does not match number of start_dates (1) to go along with them.

patterns <- data.table::data.table(state_fips = c(1,2,3),
                                   cat_origin = c('{"a": "2", "b": "3"}',
                                                  '{}',
                                                  '{"a": "4", "b": "5"}'))
> patterns
   state_fips           cat_origin
1:          1 {"a": "2", "b": "3"}
2:          2                   {}
3:          3 {"a": "4", "b": "5"}
> 
expand_cat_json(
  patterns,
  'cat_origin',
  'index',
  by = F,
  na.rm = T
)
   initial_rowno cat_origin index
1:             1          2     a
2:             1          3     b
3:             2          4     a
4:             2          5     b
expand_cat_json(
  patterns,
  'cat_origin',
  'index',
  by = F,
  na.rm = F
)
   initial_rowno cat_origin index
1:             1          2     a
2:             1          3     b
3:             3          4     a
4:             3          5     b

library(data.table)
library(SafeGraphR)
library(fst)
library(magrittr)

# Load Core POI data ----
core_poi <- read_many_csvs(dir = "/data1/safegraph/core_poi/2020/11/06/11/")

# Limit to POI that give open hours
open_hours_only <- core_poi[open_hours != ""]

convert_hour_str <- function(time_str, midnight_is_zero = TRUE) {
    # Convert an %H:%M time string to numeric, e.g., "08:15" -> 8.25
    time_POSIX <- as.POSIXlt(time_str, format = "%H:%M")
    result <- hour(time_POSIX) + minute(time_POSIX) / 60
    if (!midnight_is_zero) {
        result[result == 0] <- 24
    }
    return(result)
}

convert_JSON_hours <- function(hours_clean) {
    # Convert a JSON string listing hours open and closed into a data.table
    # hours_clean <- unique_hours$open_hours_clean[96] # DEBUG
    
    hour_list <- jsonlite::fromJSON(hours_clean) # This takes a long, long time.
    
    # Keep only non-empty
    hour_list <- hour_list[lapply(hour_list,length)>0]
    
    hour_dt <- rbindlist(lapply(hour_list, as.data.table), idcol = "dow")
    setnames(hour_dt, c("V1", "V2"), c("open", "close"))
    hour_dt[, `:=`(open = convert_hour_str(open),
                   close = convert_hour_str(close, midnight_is_zero = F))]
    hour_dt
}

expand_hours <- function(dt) {
    # dt <- open_hours_only[1:10000] # DEBUG

    # To save on parsing time, get unique values of open_hours
    unique_hours <- dt[, .N, by = open_hours] %>% .[, N := NULL]
    
    # Remove extra escaped quotes
    unique_hours[, open_hours_clean := stringr::str_replace_all(open_hours, '\\"\\"','\\"')]
    
    # Get a data.table where each obs is row-by-dow-open/close interval
    unique_hours_dt <- unique_hours[, convert_JSON_hours(open_hours_clean), by = open_hours]
    
    # Merge (M:M) back to original dataset
    dt_final <- merge(dt[, .(placekey, open_hours)], 
                      unique_hours_dt, 
                      by = "open_hours", 
                      allow.cartesian = T)
    
    dt_final <- dt_final[, .(placekey, dow, open, close)]
    dt_final
}

expanded_hours <- expand_hours(open_hours_only[sample(.N, 100)])