Bulk download of files using 'aria2c'

Bulk download of files using 'aria2c'

aria2c_bulk_donwload(
  vector_or_file_path,
  default_directory,
  user = NULL,
  password = NULL,
  threads = 1,
  verbose = FALSE,
  secondary_args_aria = "--allow-overwrite --retry-wait=5 --max-tries=0"
)

Arguments

vector_or_file_path

either a vector of character strings or a valid path to a text file. See the output of the 'aria2c_download_paths()' function for the correct format.

default_directory

a character string specifying a valid path where the files will be saved

user

either NULL or a character string specifying the 'user' (normally this is the 'username' required in specific websites to have access and download files)

password

either NULL or a character string specifying the 'password' (normally this is the 'password' required in specific websites to have access and download files)

threads

an integer value specifying the number of threads to run in parallel

verbose

a boolean. If TRUE then information will be printed out in the console

secondary_args_aria

a character vector specifying the additional parameters that can be passed to the 'aria2c' function. For instance, "--retry-wait": specifies the seconds to wait between retries and "--max-tries=0" means unlimited re-tries. See the References section for more details.

Value

a character vector based on the verbosity of the function

References

https://aria2.github.io/manual/en/html/aria2c.html

https://aria2.github.io/manual/en/html/aria2c.html#cmdoption-retry-wait

https://aria2.github.io/manual/en/html/aria2c.html#cmdoption-m

https://aria2.github.io/manual/en/html/aria2c.html#exit-status

Examples

if (FALSE) {

require(PlanetNICFI)

#....................................
# first extract the available Mosaics
#....................................

api_key = 'use_your_planet_nicfi_API_key'

mosaic_files = nicfi_mosaics(planet_api_key = api_key,
                             type = 'monthly',
                             crs_bbox = 4326,
                             URL = 'https://api.planet.com/basemaps/v1/mosaics',
                             verbose = TRUE)

#....................................
# keep the mosaic of 'September 2020'
#....................................

keep_idx = 1
mosaic_ID = mosaic_files$dtbl_mosaic$id[keep_idx]


#................................................................
# then extract the available Quad files for the Mosaic for an AOI
#................................................................

wkt_file = system.file('data_files/Sugar_Cane_Bolivia.wkt', package = "PlanetNICFI")
WKT = readLines(wkt_file, warn = FALSE)

quad_files = nicfi_quads_bbox(planet_api_key = api_key,
                              mosaic_id = mosaic_ID,
                              bbox_AOI = NULL,
                              wkt_AOI = WKT,
                              page_size = 10,
                              crs_bbox = 4326,
                              verbose = TRUE)

#..................................
# formated aria2c download weblinks
#..................................

web_links_aria2c = aria2c_download_paths(mosaic_output = mosaic_files,
                                         mosaic_id = mosaic_ID,
                                         quads_output = quad_files,
                                         img_type = 'tif')

#.........................................................
# download the .tif files that intersect with the bbox AOI
#.........................................................

temp_dir_out = tempdir()

all_threads = parallel::detectCores()
set_threads = length(web_links_aria2c) / 2
num_threads = ifelse(set_threads < all_threads, set_threads, all_threads)
aria_args = '--allow-overwrite --file-allocation=none --retry-wait=5 --max-tries=0'

res_downl = aria2c_bulk_donwload(vector_or_file_path = web_links_aria2c,
                                 default_directory = temp_dir_out,
                                 user = NULL,
                                 password = NULL,
                                 threads = num_threads,
                                 verbose = TRUE,
                                 secondary_args_aria = aria_args)
}