when large matrix or df split into list, does furrr workers only access list element? about furrr HOT 1 CLOSED

library(purrr)
library(rlang)

dim <- c(50, 1000)
mat <- array(seq_len(prod(dim)), dim = dim)
colnames(mat) <- seq_len(ncol(mat))
rownames(mat) <- seq_len(nrow(mat))

# do the splitting
data <- mat |>
  split(rep(colnames(mat), each = nrow(mat))) |>
  map(set_names, rownames(mat))

object.size(mat)
#> 267688 bytes
object.size(data)
#> 3680208 bytes

# Over 13 times larger!
unclass(object.size(data) / object.size(mat))
#> [1] 13.74812

# The problem seems to be the repeated row names, so lets try not
# adding those on
data <- mat |>
  split(rep(colnames(mat), each = nrow(mat)))

# That is more what you'd expect since you are just sharding a matrix.
# The extra ~.2 is from the memory overhead of the list
unclass(object.size(data) / object.size(mat))
#> [1] 1.196199