ropensci / ediutils Goto Github PK

# This is a list with all the package IDs I am trying to access (n=81)
package_ids <- data_list$packageid #chr

# Download all of the data entities for the package IDs in the list
for(i in 1:length(package_ids)){
  data_entity_names <- read_data_entity_names(packageId = package_ids[i])
}

Error in EDIutils::validate_path(fun.args$path) : 
  The directory specified by the argument "path" does not exist! Please enter the correct path for your dataset working directory.

#' @param affiliation
#'     (character) Affiliation corresponding with the user.id argument supplied
#'     above. Can be: 'LTER' or 'EDI'.

library (EDIutils)
library (magrittr)
library (data.table)

#assume you used the awesome workflow already to get the package you want
pkgid='knb-lter-nwt.210.1'

#get the ids
entities_id=EDIutils::api_list_data_entities(pkgid) 
#sort the urls
entities_id$url=paste0('https://pasta.lternet.edu/package/data/eml/',
                       gsub('\\.', '/', pkgid), '/', entities_id$identifier)
#also want the names
entities_name=sapply(entities_id$identifier, function(x)
  api_read_data_entity_name('knb-lter-nwt.210.1', x, 
                            environment = 'production'))%>%
  #spaces = hassle to work with, get rid of them
  gsub(' ', '_', .)

#download all
alldata=list()
#should add some EML scraping here to make it read_delim based on the delimiter
#headers TRUE/FALSE, etc
for (k in 1:length(entities_id)){
  alldata[[entities_name[k]]]=read.csv(entities_id$url[k])
}
#probably optional whether you want to bindrows them into a single df or not
#this is an example where it makes sense to but in other instances you might not
finally=alldata%>%rbindlist(., idcol=TRUE, use.names=TRUE)

# Load needed libaries
library(EDIutils)

# Clear environment
rm(list = ls())

# Define helper function
## Swaps human equivalents of wildcards for Solr wildcard
solr_wild <- function(bit){
  
  # Handle empty `bit`
  if(is.null(bit) == TRUE){
    
    # Replace with wildcard
    bit_v2 <- "*"
  }
  
  # Handle English equivalents for wildcard
  else if(length(bit) == 1){
    
    # Replace allowed keywords with wildcard
    bit_v2 <- gsub(pattern = "all|any", replacement = "*", x = bit)
  } 
  
  # If neither condition is met, return whatever was originally supplied
  else { bit_v2 <- bit }
  
  # Return finished product
  return(bit_v2) }

# Example(s)
solr_wild(bit = NULL)
solr_wild(bit = "any")
solr_wild(bit = "something else")

# Define helper function
## Parses English text into Solr syntax (i.e., right delimiters, etc.)
solrize <- function(bit){
  
  # Replace spaces with hyphens
  bit_v2 <- gsub(pattern = " ", replacement = "-", x = bit)
  
  # If more than one value, handle that
  if(length(bit_v2) > 1){
    
    # Collapse with plus signs
    bit_v3 <- paste0("(", paste0(bit_v2, collapse = "+"), ")")
    
  } else { bit_v3 <- bit_v2 }
  
  # Return finished bit
  return(bit_v3) }

# Example(s)
solrize(bit = c("primary production", "plants"))

# Define function to generate query
make_query <- function(keywords = NULL, subjects = NULL, authors = NULL, 
                       scopes = NULL, excl_scopes = NULL, 
                       return_fields = "all", limit = 10){

  ## Error Checking ----
  # Define supported return 'return_fields'
  good_fields <- c("*", "all", "abstract", "begindate", "doi", "enddate", "funding", "geographicdescription", "id", "methods", "packageid", "pubdate", "responsibleParties", "scope", "site", "taxonomic", "title", "authors", "spatialCoverage", "sources", "keywords", "organizations", "singledates", "timescales")
  
  # Error out for unsupported ones
  if(all(return_fields %in% good_fields) != TRUE)
    stop("Unrecognized return field(s): ", 
         paste(base::setdiff(x = return_fields, y = good_fields), collapse = "; "))
  
  # Error out for non-numeric limit
  if(is.numeric(limit) != TRUE){
    message("`limit` must be numeric, coercing to 10")
    limit <- 10 }
  
  ## Solr Query Construction ----
  # Make start of query object
  query_v0 <- "q="
  
  # If keywords are provided:
  ### 1. Turn into Solr Syntax
  solr_kw <- solrize(bit = solr_wild(bit = keywords)) 
  
  ### 2. Add to query
  query_v1 <- paste0(query_v0, "keyword:", solr_kw)
  
  # Handle authors
  solr_aut <- solrize(bit = solr_wild(bit = authors))
  query_v2 <- paste0(query_v1, "&fq=", "author:", solr_aut)
  
  # Handle subjects
  solr_sub <- solrize(bit = solr_wild(bit = subjects))
  query_v3 <- paste0(query_v2, "&fq=", "subject:", solr_sub)
  
  # Handle scopes
  solr_scp <- solrize(bit = solr_wild(bit = scopes))
  query_v4 <- paste0(query_v3, "&fq=", "scope:", solr_scp)
  
  # EXCLUDED scopes
  ## Handled differently because don't want to swap `NULL` for wildcard
  if(is.null(excl_scopes) != TRUE){
    
    # Solr-ize
    solr_excl_scp <- solrize(bit = excl_scopes)
    
    # Add to query
    query_v5 <- paste0(query_v4, "&fq=", "-scope:", solr_excl_scp)
    
    # Or skip
  } else { query_v5 <- query_v4 }
  
  # Parse return fields
  ## Solr syntax for multiple entries differs here from other elements of query
  solr_fl <- paste(solr_wild(bit = return_fields), collapse=",")
  query_v6 <- paste0(query_v5, "&fl=", solr_fl)
  
  # Finally, assemble full query with row limit
  solr_query <- paste0(query_v6, "&rows=", limit)
  
  # Return that to the user
  return(solr_query) }

#  Invoke function
( request <- make_query(keywords = "*", 
                        scopes = "knb-lter-fce",
                        excl_scopes = c("ecotrends", "lter landsat"),
                        return_fields =  c("title", "authors", "id", "doi"),
                        limit = 10) )

# Test assembled query
EDIutils::search_data_packages(query = request)

# Test use of `make_query` inside of `search_data_packages`
EDIutils::search_data_packages(query = make_query(excl_scopes = "knb-lter-fce",
                                                  return_fields = c("title", "id")))

library(rvest)
library(EDIutils)
library(EML)
library(dplyr)
library(stringr)

url <- "https://doi.org/10.6073/pasta/31b32868ddbb099c4b5480fb00eb2481"

landingPage <- read_html(url)

pageSubset <- landingPage %>%
  html_nodes(".no-list-style") %>%
  html_text()

packageId <- str_extract(grep("knb-lter-", pageSubset, value = TRUE)[[1]], "^\\S*")

packageProv <- emld::as_emld(EDIutils::api_get_provenance_metadata(packageId))
packageProv$`@context` <- NULL
packageProv$`@type` <- NULL

# desired output
packageProv 

Templating table attributes ...
Warning: Error in detect_errors: occurrence.txt contains an inconsistent number of field delimeters. The correct number of field delimiters for this table appears to be 237. Deviation from this occurs at rows: 141, 209, 520, 1056, 1058, 1059, 1071, 1119, 1120, 1121, 1132, 1140, 1142, 1147, 1153, 1157, 1165, 1185, 1189, 1219, 1238, 3852, 3932 ... Check the number of field delimiters in these rows. All rows of your table must contain a consistent number of fields.

Templating taxonomic coverage ...
Error in EDIutils::validate_file_names(path = fun.args$data.path, data.files = fun.args$taxa.table) : 
  Invalid data.files entered: /home/pndb-elie/dataPackagesOutput/emlAssemblyLine/test_emldp/decomp.csv

sum(use_i) == length(data.files)

length(which(sum(use_i))) == length(data.files)

library(EDIutils)

query <- search_data_packages(query = 'q=scope:(knb-lter-fce)&fl=doi,title,packageid,begindate,enddate,pubdate')

> new_doi
[1] "edi.416.6"
> EDIutils::api_update_data_package(
+     path = eml_path,
+     package.id = new_doi,
+     environment = "staging",
+     user.id = usern,
+     user.pass = passw,
+     affiliation = "EDI"
+ )
Error in open.connection(con, "rb") : HTTP error 401.
> traceback()
7: open.connection(con, "rb")
6: open(con, "rb")
5: read_connection(path)
4: datasource_connection(file, skip, skip_empty_rows, comment, skip_quote)
3: datasource(file, skip_empty_rows = FALSE)
2: readr::read_file(paste0(url_env(environment), ".lternet.edu/package/report/eml/", 
       stringr::str_replace_all(package.id, "\\.", "/")))
1: EDIutils::api_update_data_package(path = eml_path, package.id = new_doi, 
       environment = "staging", user.id = usern, user.pass = passw, 
       affiliation = "EDI")
> sessionInfo()
R version 4.0.4 (2021-02-15)
Platform: x86_64-w64-mingw32/x64 (64-bit)
Running under: Windows 10 x64 (build 19042)