Currently works when separated by a comma but not & or |. Eg.:

An example of how to use the bcgov packages together in a work flow?

library(bcdata) 
  bcdc_search("wildfire")
#> $`fire-locations-current`
#> $sector
#> [1] "Natural Resources"
#> 
#> $preview_zoom_level
#> [1] "9"
#> 
#> $maintainer
#> NULL
#> 
SNIP
#> 
#> attr(,"class")
#> [1] "bcdc_record"
#> 
#> attr(,"class")
#> [1] "bcdc_recordlist"

^{Created on 2019-01-29 by the reprex package (v0.2.1)}

Or something similar.

Here's a reproducible example

library(bcdata)

pori <- bcdc_query_geodata("freshwater-atlas-stream-network") %>%
  filter(WATERSHED_GROUP_CODE %in% "PORI") %>%
  collect()

# fails
x <- bcdc_query_geodata("freshwater-atlas-stream-network") %>%
  filter(WATERSHED_KEY %in% pori$WATERSHED_KEY) %>%
  collect()

# fails with different error
x <- bcdc_query_geodata("freshwater-atlas-stream-network") %>%
  filter(WATERSHED_KEY %in% pori$WATERSHED_KEY[1:510]) %>%
  collect()

# works
x <- bcdc_query_geodata("freshwater-atlas-stream-network") %>%
  filter(WATERSHED_KEY %in% pori$WATERSHED_KEY[1:509]) %>%
  collect()

─ Session info ─────────────────────────────────────────────────────────────────────
 setting  value                       
 version  R version 3.5.3 (2019-03-11)
 os       macOS High Sierra 10.13.6   
 system   x86_64, darwin15.6.0        
 ui       RStudio                     
 language (EN)                        
 collate  en_CA.UTF-8                 
 ctype    en_CA.UTF-8                 
 tz       America/Los_Angeles         
 date     2019-05-01                  

─ Packages ─────────────────────────────────────────────────────────────────────────
 package     * version    date       lib source                       
 assertthat    0.2.1      2019-03-21 [1] CRAN (R 3.5.3)               
 backports     1.1.4      2019-04-10 [1] CRAN (R 3.5.3)               
 bcdata      * 0.0.0.9000 2019-04-29 [1] Github (bcgov/bcdata@7591a95)
 callr         3.2.0      2019-03-15 [1] CRAN (R 3.5.2)               
 class         7.3-15     2019-01-01 [1] CRAN (R 3.5.3)               
 classInt      0.3-1      2018-12-18 [1] CRAN (R 3.5.0)               
 cli           1.1.0      2019-03-19 [1] CRAN (R 3.5.3)               
 colorspace    1.4-1      2019-03-18 [1] CRAN (R 3.5.2)               
 crayon        1.3.4      2017-09-16 [1] CRAN (R 3.5.0)               
 crul          0.7.4      2019-03-28 [1] CRAN (R 3.5.3)               
 curl          3.3        2019-01-10 [1] CRAN (R 3.5.2)               
 DBI           1.0.0      2018-05-02 [1] CRAN (R 3.5.0)               
 dbplyr        1.3.0      2019-01-09 [1] CRAN (R 3.5.2)               
 desc          1.2.0      2018-05-01 [1] CRAN (R 3.5.0)               
 devtools    * 2.0.2      2019-04-08 [1] CRAN (R 3.5.2)               
 digest        0.6.18     2018-10-10 [1] CRAN (R 3.5.0)               
 dplyr         0.8.0.1    2019-02-15 [1] CRAN (R 3.5.2)               
 drat        * 0.1.5      2019-03-28 [1] CRAN (R 3.5.3)               
 e1071         1.7-1      2019-03-19 [1] CRAN (R 3.5.2)               
 fs            1.2.7      2019-03-19 [1] CRAN (R 3.5.2)               
 ggplot2       3.1.1      2019-04-07 [1] CRAN (R 3.5.2)               
 glue          1.3.1      2019-03-12 [1] CRAN (R 3.5.3)               
 gtable        0.3.0      2019-03-25 [1] CRAN (R 3.5.3)               
 httpcode      0.2.0      2016-11-14 [1] CRAN (R 3.5.0)               
 httr          1.4.0      2018-12-11 [1] CRAN (R 3.5.1)               
 jsonlite      1.6        2018-12-07 [1] CRAN (R 3.5.0)               
 lazyeval      0.2.2      2019-03-15 [1] CRAN (R 3.5.2)               
 magrittr      1.5        2014-11-22 [1] CRAN (R 3.5.0)               
 memoise       1.1.0      2017-04-21 [1] CRAN (R 3.5.0)               
 munsell       0.5.0      2018-06-12 [1] CRAN (R 3.5.0)               
 packrat       0.5.0      2018-11-14 [1] CRAN (R 3.5.1)               
 pillar        1.3.1      2018-12-15 [1] CRAN (R 3.5.0)               
 pkgbuild      1.0.3      2019-03-20 [1] CRAN (R 3.5.3)               
 pkgconfig     2.0.2      2018-08-16 [1] CRAN (R 3.5.0)               
 pkgload       1.0.2      2018-10-29 [1] CRAN (R 3.5.0)               
 plyr          1.8.4      2016-06-08 [1] CRAN (R 3.5.0)               
 prettyunits   1.0.2      2015-07-13 [1] CRAN (R 3.5.0)               
 processx      3.3.0      2019-03-10 [1] CRAN (R 3.5.2)               
 ps            1.3.0      2018-12-21 [1] CRAN (R 3.5.0)               
 purrr         0.3.2      2019-03-15 [1] CRAN (R 3.5.2)               
 R6            2.4.0      2019-02-14 [1] CRAN (R 3.5.2)               
 Rcpp          1.0.1      2019-03-17 [1] CRAN (R 3.5.2)               
 remotes       2.0.4      2019-04-10 [1] CRAN (R 3.5.3)               
 rlang         0.3.4      2019-04-07 [1] CRAN (R 3.5.2)               
 rprojroot     1.3-2      2018-01-03 [1] CRAN (R 3.5.0)               
 rstudioapi    0.10       2019-03-19 [1] CRAN (R 3.5.3)               
 scales        1.0.0      2018-08-09 [1] CRAN (R 3.5.0)               
 sessioninfo   1.1.1      2018-11-05 [1] CRAN (R 3.5.0)               
 sf            0.7-3      2019-02-21 [1] CRAN (R 3.5.2)               
 slackr      * 1.4.2      2016-07-20 [1] CRAN (R 3.5.0)               
 testthat    * 2.0.1      2018-10-13 [1] CRAN (R 3.5.0)               
 tibble        2.1.1      2019-03-16 [1] CRAN (R 3.5.2)               
 tidyselect    0.2.5      2018-10-11 [1] CRAN (R 3.5.0)               
 triebeard     0.3.0      2016-08-04 [1] CRAN (R 3.5.0)               
 units         0.6-2      2018-12-05 [1] CRAN (R 3.5.0)               
 urltools      1.7.3      2019-04-14 [1] CRAN (R 3.5.2)               
 usethis     * 1.5.0      2019-04-07 [1] CRAN (R 3.5.2)               
 withr         2.1.2      2018-03-15 [1] CRAN (R 3.5.0)               

[1] /Library/Frameworks/R.framework/Versions/3.5/Resources/library
> 

Allow terms to be passed as a vector, and allow use of | for OR

If user gives bcdc_get_geodata a record that is not WFS then use bcdc_get_data and method within.

• for getting datasets from bcdc data or file store

Find endpoint that gives number of points matched without having to download the whole wfs layer then paginate over the large file.

The idea here is to mimic the dbplyr flow of delayed evaluation of queries. bcdc_get_geodata() will return an object of class bcdc_promise with all the information required to send a query to the wfs service, but hasn't actually done it yet. A print.bcdc_promise method will print metadata about the record (column names, types, etc). In order to actually get the result, a collect.bcdc_promise method will be used. The resulting sf object will also have a bcdc_sf class appended to it, with attributes such as url, sql string, etc.

bcdc_get_geodata("bc-airports") %>%
    collect()

The current query structure that is currently passed to the ... in bcdc_get_geodata will move to a filter method (#31), and a select method will also be developed (#37). Each of these will accept a bcdc_promise object, and return a bcdc_promise object with a modified wfs query. As outlined above, the collect() method will be used to send the query and return the resulting sf object.

As it is also probably desirable to preview the data visually before (or after) collecting, a preview (name TBD) function with methods for both bcdc_promise and bcdc_sf will use the WMS service to show a leaflet map (see also #15 and this gist).

Some utility functions such as show_call.bcdc_promise() and show_call.bcdc_sf() would also be helpful.

cc @boshek @j-galloway @stephhazlitt

should work with
bcdc_get_data("record", format = NULL, ...)
rather than `bcdc_get_data("resource")

the ... will pass arguments to the read function (e.g., readr::read_csv(x, ...) or sf::st_read(x, ...)

If format = NULL and it's a wfs record (resource_storage_location = "BCGW Data Store") just download it. If not wfs and interactive() then show a menu of formats to choose from, otherwise fail.

Could also have a read_fun argument in bcdc_get_data that provides a custom read function.

Also use outputFormat=csv

$ curl "https://openmaps.gov.bc.ca/geo/pub/wfs?SERVICE=WFS&VERSION=2.0.0&REQUEST=GetFeature&outputFormat=csv&typeNames=WHSE_ENVIRONMENTAL_MONITORING.EMS_MONITORING_LOCN_GROUPS_SVW&propertyName=FEATURE_CODE" | head
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0FID,MONITORING_GROUP_ID,GROUP_NAME,OFFICE_CD,AEMS_TAG,FEATURE_CODE
WHSE_ENVIRONMENTAL_MONITORING.EMS_MONITORING_LOCN_GROUPS_SVW.fid--120209e6_1694abefcd7_13b7,330,HOWE SOUND AREA,20,0300189,EP10020300
WHSE_ENVIRONMENTAL_MONITORING.EMS_MONITORING_LOCN_GROUPS_SVW.fid--120209e6_1694abefcd7_13b8,330,HOWE SOUND AREA,20,0300190,EP10020300
WHSE_ENVIRONMENTAL_MONITORING.EMS_MONITORING_LOCN_GROUPS_SVW.fid--120209e6_1694abefcd7_13b9,315,ABB-SUMAS AQUIFER WELLS,20,0301012,WA12100190
WHSE_ENVIRONMENTAL_MONITORING.EMS_MONITORING_LOCN_GROUPS_SVW.fid--120209e6_1694abefcd7_13ba,357,BURRARD INLET PROJECT,20,0301046,EP10020300
WHSE_ENVIRONMENTAL_MONITORING.EMS_MONITORING_LOCN_GROUPS_SVW.fid--120209e6_1694abefcd7_13bb,217,BI TEST3,20,0301046,EP10020300
WHSE_ENVIRONMENTAL_MONITORING.EMS_MONITORING_LOCN_GROUPS_SVW.fid--120209e6_1694abefcd7_13bc,357,BURRARD INLET PROJECT,20,0301047,EP10020300
WHSE_ENVIRONMENTAL_MONITORING.EMS_MONITORING_LOCN_GROUPS_SVW.fid--120209e6_1694abefcd7_13bd,217,BI TEST3,20,0301047,EP10020300
WHSE_ENVIRONMENTAL_MONITORING.EMS_MONITORING_LOCN_GROUPS_SVW.fid--120209e6_1694abefcd7_13be,357,BURRARD INLET PROJECT,20,0301048,EP10020300
WHSE_ENVIRONMENTAL_MONITORING.EMS_MONITORING_LOCN_GROUPS_SVW.fid--120209e6_1694abefcd7_13bf,217,BI TEST3,20,0301048,EP10020300
100 32768    0 32768    0     0   182k      0 --:--:-- --:--:-- --:--:--  182k

Something like this could work:

  if(!curl::has_internet()){
    return(message("A working internet connection is required to download and import the climate indices."))
  }

Using describe_feature generate a glimpse method from dplyr so that users can view columns and types.

Before Release:

• close v0.1.0 labelled issues
• update README (new API, add sticker)
• bcdata Getting Started vignette
• go through function documentation
• finish bcdc_query_geodata() vignette (?)
• pkgdown site
• fix README licence text
• finalize hex sticker
• edit NEWS.md
• bump version in DESCRIPTION
• rebuild pkgdown
• tag release

In print.bcdc_get_record change formatting of wfs resources:

Name: WFS request (Spatial Data) # also try to get info on the spatial type from the record
format: wfs

Originally posted by @ateucher in #54 (comment)

Eg. ski-resorts:

curl "https://openmaps.gov.bc.ca/geo/pubST=DescribeFeatureType&outputFormat=application/json&typeNames=WHSE_IMAGERY_AND_BASE_MAPS.GSR_SKI_RESORTS_SV"

This has implications for the CQL constructors (e.g., WITHIN)

Use propertName=attribute1,attribute2,geometry_column_name

PR #64 sets a length of the CQL_FILTER limit. From that it should be possible to determine if creating a bounding box for a spatial predicate is required or not.

You can get a record with both the record id (bcdc_get_record("76b1b7a3-2112-4444-857a-afccf7b20da8") and the record name (bcdc_get_record("bc-airports"). Users should get a warning in the console if they use the name -- while fine interactively, this is a more fragile approach in scripts as record names can change (with no redirects). Record IDs do not change.

• Use crul package
• Combine bcdc_map and bcdc_wms

We can leverage the COUNT parameter of wfs adding it to the query like:

c(x$query_list, COUNT = 10)

top_n is not defined as a generic in dplyr so then maybe we should simply use head.

This could do much of the heavy lifting of various file formats for us:

https://github.com/leeper/rio

Particularly since it basically does already what this is doing:

Nice package guys. Thanks!

Everything seems to function for me except the golden boy "bcdc_get_geodata" function. It returns returns "Error in FUN(X[[i]], ...) : subscript out of bounds". Guessing it is some sort of issue on my end but thought I'd throw it at you in case you have seen it before.

P.S. I have managed to write a script to get the job done (for files with less than 10000 features for now) but I bet yours is far cleaner and I would rather ride your coattails.....

My apologies if this issue is annoying due to my novice status. This is my first time posting on github.

library(bcdata)
#> 
#> Attaching package: 'bcdata'
#> The following object is masked from 'package:stats':
#> 
#>     filter

x <- c("airport", "heliport")
bcdc_query_geodata("bc-airports") %>% 
  filter(DESCRIPTION == x[1]) %>% 
  show_query()
#> <url> 
#> https://openmaps.gov.bc.ca/geo/pub/wfs?
#> SERVICE=WFS&
#> VERSION=2.0.0&
#> REQUEST=GetFeature&
#> outputFormat=application%2Fjson&
#> typeNames=WHSE_IMAGERY_AND_BASE_MAPS.GSR_AIRPORTS_SVW&
#> SRSNAME=EPSG%3A3005
#> <SQL> 
#> ("DESCRIPTION" = CASE WHEN (1.0) THEN (('airport', 'heliport')) END)

y <- data.frame(type = c("airport", "heliport"))
bcdc_query_geodata("bc-airports") %>% 
  filter(DESCRIPTION %in% y$type) %>% 
  show_query()
#> Error in UseMethod("escape"): no applicable method for 'escape' applied to an object of class "data.frame"

^{Created on 2019-05-02 by the reprex package (v0.2.1)}

I would like to pass an object to filter.bcdc_promise() from with my own function.

filter_wskey -> function(x){
   bcdata::bcdc_query_geodata("freshwater-atlas-stream-network”) %>%
   	filter(WATERSHED_KEY %in% x) %>%
   	collect()
}
filter_wskey(c(360709847, 360843586))

Any suggestions?

bcdc_query_geodata("bc-major-cities-points-1-2-000-000-digital-baseline-mapping") %>% 
 select(geometry)
# A B.C. Data Catalogue Record: NA records and 11 columns
# Columns:
# A tibble: 11 x 4
   col_name   selectable remote_col_type          local_col_type
   <chr>      <lgl>      <chr>                    <chr>         
 1 id         FALSE      xsd:string               character     
 2 FCODE      FALSE      xsd:string               character     
 3 BCMJ_TAG   FALSE      xsd:decimal              numeric       
 4 NAME       FALSE      xsd:string               character     
 5 CITY_TYPE  FALSE      xsd:string               character     
 6 LONG_TYPE  FALSE      xsd:string               character     
 7 POP_2000   FALSE      xsd:decimal              numeric       
 8 POP_SOURCE FALSE      xsd:string               character     
 9 OBJECTID   FALSE      xsd:decimal              numeric       
10 geometry   TRUE       gml:PointPropertyType    sfc geometry  
11 geometry   TRUE       gml:GeometryPropertyType sfc geometry  

Fixed in this commit for describe_feature: 0410988

• bcdc_get_record()
• bcdc_facets()
• bcdc_list()
• bcdc_search()

Mostly done, but the ability to get the call that you used to get the final sf object could be useful. Moving to 0.2.0

Originally posted by @ateucher in #39 (comment)

Is there an elegant way to read the column names of a dataset, and perhaps the levels of the data within a column before downloading a dataset? I am thinking that this would be useful for particularly large datasets (e.g. harvest layer, etc).

Here is an example:
fires = bcdc_get_geodata(22c7cb44-1463-48f7-8e47-88857f207702, query = "FIRE_YEAR==2017")

How can I know what the "FIRE_YEAR" column name is through R, and the years available?

Thanks! Great package.

E.g., intersect two wfs layers

library(bcdata)
foo <- bcdc_search(res_format = "wms", n = 1000)
#> Found 653 matches. Returning the first 1000.
#> To see them all, rerun the search and set the 'n' argument to 653.
bar <- foo[!unname(sapply(foo, function(x) sum(grepl("SDO_GEOMETRY", x$details$data_type))))]
#> Warning: Unknown or uninitialised column: 'data_type'.

#> Warning: Unknown or uninitialised column: 'data_type'.

#> Warning: Unknown or uninitialised column: 'data_type'.

#> Warning: Unknown or uninitialised column: 'data_type'.

#> Warning: Unknown or uninitialised column: 'data_type'.

#> Warning: Unknown or uninitialised column: 'data_type'.

#> Warning: Unknown or uninitialised column: 'data_type'.
lapply(bar, function(x) paste0("https://catalogue.data.gov.bc.ca/dataset/", x$id))
#> $`imagery-web-map-service`
#> [1] "https://catalogue.data.gov.bc.ca/dataset/73b2a989-df6b-4298-bd38-c60f0254a453"
#> 
#> $`imagery-spot-15m-web-map-service`
#> [1] "https://catalogue.data.gov.bc.ca/dataset/bcad8f9e-274b-4a29-9e06-d05def3a50c2"
#> 
#> $`landsat-web-map-service`
#> [1] "https://catalogue.data.gov.bc.ca/dataset/ac0a7aa9-f56d-45b1-bd17-8554843de6a5"
#> 
#> $`orthophoto-web-map-service`
#> [1] "https://catalogue.data.gov.bc.ca/dataset/3fa96b60-0260-45b7-b231-3ce4610852c0"
#> 
#> $`bc-roads-map-service-web-mercator`
#> [1] "https://catalogue.data.gov.bc.ca/dataset/0df78c0f-ce9c-4fa4-a741-ffbc8930d82f"
#> 
#> $`bc-web-map-library`
#> [1] "https://catalogue.data.gov.bc.ca/dataset/6164a2af-d3ac-4e92-8dbe-51a93bb5e24b"
#> 
#> $`historic-city-of-vancouver-fire-insurance-map`
#> [1] "https://catalogue.data.gov.bc.ca/dataset/1dbc0b94-54c3-4d6b-a55f-340673a5056f"

^{Created on 2019-04-01 by the reprex package (v0.2.1)}

These work:

# non-spatial filter statements separated by comma (or single)
bcdc_get_geodata("bc-major-cities-points-1-2-000-000-digital-baseline-mapping", 
                 LONG_TYPE == 'VILLAGE')
bcdc_get_geodata("bc-major-cities-points-1-2-000-000-digital-baseline-mapping", 
                 LONG_TYPE %in%  c('CITY', 'VILLAGE'))
bcdc_get_geodata("bc-major-cities-points-1-2-000-000-digital-baseline-mapping", 
                 LONG_TYPE == 'VILLAGE', POP_2000 < 1000)

# non-spatial filter statements separated by & or |
bcdc_get_geodata("bc-major-cities-points-1-2-000-000-digital-baseline-mapping", 
                 LONG_TYPE == 'VILLAGE' | POP_2000 < 1000)
bcdc_get_geodata("bc-major-cities-points-1-2-000-000-digital-baseline-mapping", 
                 LONG_TYPE == 'VILLAGE' & POP_2000 < 1000)

# spatial query:
kootenay_park <- bcdc_get_geodata("national-parks-of-canada-within-british-columbia",
                                  ENGLISH_NAME == 'Kootenay National Park of Canada')

bcdc_get_geodata("bc-major-cities-points-1-2-000-000-digital-baseline-mapping", 
                 WITHIN(kootenay_park))

# combine spatial with non-spatial separated by a comma:
bcdc_get_geodata("bc-major-cities-points-1-2-000-000-digital-baseline-mapping", 
                 WITHIN(kootenay_park), POP_2000 < 1000)

but this bonks:

# combine spatial and non-spatial separate by & or |
bcdc_get_geodata("bc-major-cities-points-1-2-000-000-digital-baseline-mapping", 
                               WITHIN(kootenay_park) | POP_2000 < 1000)

Originally posted by @ateucher in #43 (comment)

Put spatial filter expressions into a filter.bcdc_promise() method instead of in bcdc_get_geodata()

Is there a way to see the progress of a download? I would like to use this to grab some larger datasets from bcdc, the conduct analysis locally. Some files are large and I don't have any indication of the progress.

I looked at the source code of bcdc_get_geodata() but can't figure out a way to do this.

For example, the following dataset is very large, and takes a while to download (~10 minutes).

bcdc_get_geodata("b1b647a6-f271-42e0-9cd0-89ec24bce9f7")